skippkg-finder.py

#!/usr/bin/python3

import argparse
import logging
import sys


import re
from lxml import etree as ET

import osc.conf
import osc.core
from osc.core import http_GET
from osc.core import makeurl
import osclib
from osclib.core import source_file_ensure
from osclib.core import source_file_load
from osclib.conf import Config

SUPPORTED_ARCHS = ['x86_64', 'i586', 'aarch64', 'ppc64le', 's390x']
DEFAULT_REPOSITORY = 'standard'
META_PACKAGE = '000productcompose'
PRODUCTCOMPOSE_SEPERATOR_LINE = '# The following is generated by skippkg-finder'


class SkippkgFinder(object):
    def __init__(self, opensuse_project, print_only):
        self.opensuse_project = opensuse_project
        self.print_only = print_only
        self.apiurl = osc.conf.config['apiurl']
        self.debug = osc.conf.config['debug']

        config = Config.get(self.apiurl, opensuse_project)

        # Product compose input file
        self.skippkg_finder_productcompose_input_file = config.get('skippkg-finder-productcompose-input-file', '')
        # Repositories be used for product build
        self.skippkg_finder_product_repos = set(config.get('skippkg-finder-product-repos', '').split())
        # Binary package matches the regex from `skippkg-finder-skiplist-ignores`
        # be found in `package_binaries` will be ignored
        # The format must like SUSE:SLFO:Main:Build_gettext-runtime or SUSE:SLFO:Main:Build_gettext-runtime:mini
        # format definition: PROJECT-NAME_PACKAGE-NAME
        self.skiplist_to_ignore = set(config.get('skippkg-finder-to-ignore', '').split())
        # Supplement binary package to be ignored in the ftp-tree
        self.skiplist_supplement_regex = set(config.get('skippkg-finder-supplement-regex', '').split())
        # Drops off binary package from the ignoring list
        self.skiplist_ignorelist_whitelist = set(config.get('skippkg-finder-ignorelist-whitelist', '').split())

    def get_project_binary_list(self, project, repository, arch, package_binaries={}):
        """
        Return binarylist of a project
        """
        path = ['build', project, repository, arch]
        url = makeurl(self.apiurl, path, {'view': 'binaryversions'})
        root = ET.parse(http_GET(url)).getroot()

        for binary_list in root:
            package = binary_list.get('package')
            index = project + "_" + package

            if index not in package_binaries:
                package_binaries[index] = []
            for binary in binary_list:
                filename = binary.get('name')
                result = re.match(osclib.core.RPM_REGEX, filename)
                if not result:
                    continue

                if result.group('arch') == 'src' or result.group('arch') == 'nosrc':
                    continue
                if result.group('name').endswith('-debuginfo') or result.group('name').endswith('-debuginfo-32bit'):
                    continue
                if result.group('name').endswith('-debugsource'):
                    continue

                if result.group('name') not in package_binaries[index]:
                    package_binaries[index].append(result.group('name'))

        return package_binaries

    def create_output(self, input_file, to_ignore_list=[]):
        """
        Return a result data with unneeded group per pre-defined rules
        """
        output = []
        lines = input_file.splitlines()
        for line in lines:
            if line.startswith(PRODUCTCOMPOSE_SEPERATOR_LINE):
                break
            output.append(line + '\n')
        output.append(f"{PRODUCTCOMPOSE_SEPERATOR_LINE}\n")
        # Adding unneeded part
        output.append('\n')
        output.append('- name: unneeded\n')
        output.append('  packages:\n')
        for pkg in sorted(to_ignore_list):
            output.append(f"  - {pkg}\n")
        output.append('\n')
        # Adding main part
        output.append('- name: main\n')
        output.append('  add:\n')
        output.append('  - __all__\n')
        output.append('  sub:\n')
        output.append('  - unneeded\n')
        output.append('\n')

        return output

    def crawl(self):
        """Main method"""

        if not (self.skippkg_finder_product_repos and
                self.skippkg_finder_productcompose_input_file):
            print('Please define product repository list by \'skippkg-finder-product-repos\''
                  ' and \'skippkg-finder-productcompose-input-file\' in OSRT:Config')
            quit()
        # Any existing binary package from listed repositories
        fullbinarylist = []
        # package_binaries[] is a pre-formated binarylist per each package
        # Access to the conotent uses for example package_binaries['SUSE:SLFO:Main:Build_curl:mini']
        package_binaries = {}

        product_repos = {}
        for reponame in self.skippkg_finder_product_repos:
            prj, repo = reponame.split("_", 1)
            product_repos[prj] = repo
        # Inject binarylist to a list per package name no matter what archtectures was
        for arch in SUPPORTED_ARCHS:
            for prj in product_repos.keys():
                package_binaries = self.get_project_binary_list(prj, product_repos[prj], arch, package_binaries)

        for pkg in package_binaries.keys():
            fullbinarylist += package_binaries[pkg]

        # Preparing a packagelist to be ignored
        to_ignore_list = []
        for pkg in self.skiplist_to_ignore:
            if pkg in package_binaries:
                [to_ignore_list.append(p.strip()) for p in package_binaries[pkg]]
            else:
                logging.info(f"Can not find source package: {pkg}")
        for regex in self.skiplist_supplement_regex:
            for binary in fullbinarylist:
                result = re.match(regex, binary)
                if result and binary not in to_ignore_list:
                    to_ignore_list.append(binary)
        # Handling package whitelist
        [to_ignore_list.remove(pkg) for pkg in self.skiplist_ignorelist_whitelist if pkg in to_ignore_list]

        input_file = source_file_load(self.apiurl, self.opensuse_project, META_PACKAGE,
                                      str(self.skippkg_finder_productcompose_input_file).strip())
        result_string = ''.join(map(str, self.create_output(input_file, to_ignore_list)))

        if not self.print_only:
            source_file_ensure(self.apiurl, self.opensuse_project, META_PACKAGE,
                               str(self.skippkg_finder_productcompose_input_file).strip().replace('.in', ''),
                               result_string, 'Update the skip list')
        else:
            print(result_string)


def main(args):
    osc.conf.get_config(override_apiurl=args.apiurl)
    osc.conf.config['debug'] = args.debug

    if args.opensuse_project is None:
        print("Please pass --opensuse-project argument. See usage with --help.")
        quit()

    uc = SkippkgFinder(args.opensuse_project, args.print_only)
    uc.crawl()


if __name__ == '__main__':
    description = 'Overwrites unneeded part in productcompose according to the pre-defined rules. '\
                  'This tool only works for product-composer with ftp-tree build scenario.'
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL')
    parser.add_argument('-d', '--debug', action='store_true',
                        help='print info useful for debuging')
    parser.add_argument('-o', '--opensuse-project', dest='opensuse_project', metavar='OPENSUSE_PROJECT',
                        help='openSUSE project on buildservice')
    parser.add_argument('-p', '--print-only', action='store_true',
                        help='show the result instead of the uploading')

    args = parser.parse_args()

    logging.basicConfig(level=logging.DEBUG if args.debug
                        else logging.INFO)

    sys.exit(main(args))