build.py

#!/usr/bin/env python3

#  Copyright 2021 Google, Inc.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
""" Build BT targets on the host system.

For building, you will first have to stage a platform directory that has the
following structure:
|-common-mk
|-bt
|-external
|-|-rust
|-|-|-vendor

The simplest way to do this is to check out platform2 to another directory (that
is not a subdir of this bt directory), symlink bt there and symlink the rust
vendor repository as well.
"""
import argparse
import multiprocessing
import os
import platform
import shutil
import six
import subprocess
import sys
import tarfile
import time

# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
COMMON_MK_USES = [
    'asan',
    'coverage',
    'cros_host',
    'cros_debug',
    'floss_rootcanal',
    'function_elimination_experiment',
    'fuzzer',
    'fuzzer',
    'lto_experiment',
    'msan',
    'profiling',
    'proto_force_optimize_speed',
    'tcmalloc',
    'test',
    'ubsan',
]

# Use a specific commit version for common-mk to avoid build surprises.
COMMON_MK_COMMIT = "d014d561eaf5ece08166edd98b10c145ef81312d"

# Default use flags.
USE_DEFAULTS = {
    'android': False,
    'bt_nonstandard_codecs': False,
    'test': False,
}

VALID_TARGETS = [
    'all',  # All targets except test and clean
    'bloat',  # Check bloat of crates
    'clean',  # Clean up output directory
    'docs',  # Build Rust docs
    'hosttools',  # Build the host tools (i.e. packetgen)
    'main',  # Build the main C++ codebase
    'prepare',  # Prepare the output directory (gn gen + rust setup)
    'rust',  # Build only the rust components + copy artifacts to output dir
    'test',  # Run the unit tests
    'clippy',  # Run cargo clippy
    'utils',  # Build Floss utils
]

# TODO(b/190750167) - Host tests are disabled until we are full bazel build
HOST_TESTS = [
    # 'bluetooth_test_common',
    # 'bluetoothtbd_test',
    # 'net_test_avrcp',
    # 'net_test_btcore',
    # 'net_test_types',
    # 'net_test_btm_iso',
    # 'net_test_btpackets',
]

# Map of git repos to bootstrap and what commit to check them out at. None
# values will just checkout to HEAD.
BOOTSTRAP_GIT_REPOS = {
    'platform2': ('https://chromium.googlesource.com/chromiumos/platform2', COMMON_MK_COMMIT),
    'rust_crates': ('https://chromium.googlesource.com/chromiumos/third_party/rust_crates', None),
    'proto_logging': ('https://android.googlesource.com/platform/frameworks/proto_logging', None),
}

# List of packages required for linux build
REQUIRED_APT_PACKAGES = [
    'bison',
    'build-essential',
    'curl',
    'debmake',
    'flatbuffers-compiler',
    'flex',
    'g++-multilib',
    'gcc-multilib',
    'generate-ninja',
    'gnupg',
    'gperf',
    'libabsl-dev',
    'libc++abi-dev',
    'libc++-dev',
    'libdbus-1-dev',
    'libdouble-conversion-dev',
    'libevent-dev',
    'libevent-dev',
    'libflatbuffers-dev',
    'libfmt-dev',
    'libgl1-mesa-dev',
    'libglib2.0-dev',
    'libgtest-dev',
    'libgmock-dev',
    'liblc3-dev',
    'liblz4-tool',
    'libncurses5',
    'libnss3-dev',
    'libfmt-dev',
    'libprotobuf-dev',
    'libre2-9',
    'libre2-dev',
    'libssl-dev',
    'libtinyxml2-dev',
    'libx11-dev',
    'libxml2-utils',
    'ninja-build',
    'openssl',
    'protobuf-compiler',
    'unzip',
    'x11proto-core-dev',
    'xsltproc',
    'zip',
    'zlib1g-dev',
]

# List of cargo packages required for linux build
REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd', 'pdl-compiler', 'grpcio-compiler', 'cargo-bloat']

APT_PKG_LIST = ['apt', '-qq', 'list']
CARGO_PKG_LIST = ['cargo', 'install', '--list']


class UseFlags():

    def __init__(self, use_flags):
        """ Construct the use flags.

        Args:
            use_flags: List of use flags parsed from the command.
        """
        self.flags = {}

        # Import use flags required by common-mk
        for use in COMMON_MK_USES:
            self.set_flag(use, False)

        # Set our defaults
        for use, value in USE_DEFAULTS.items():
            self.set_flag(use, value)

        # Set use flags - value is set to True unless the use starts with -
        # All given use flags always override the defaults
        for use in use_flags:
            value = not use.startswith('-')
            self.set_flag(use, value)

    def set_flag(self, key, value=True):
        setattr(self, key, value)
        self.flags[key] = value


class HostBuild():

    def __init__(self, args):
        """ Construct the builder.

        Args:
            args: Parsed arguments from ArgumentParser
        """
        self.args = args

        # Set jobs to number of cpus unless explicitly set
        self.jobs = self.args.jobs
        if not self.jobs:
            self.jobs = multiprocessing.cpu_count()
            sys.stderr.write("Number of jobs = {}\n".format(self.jobs))

        # Normalize bootstrap dir and make sure it exists
        self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
        os.makedirs(self.bootstrap_dir, exist_ok=True)

        # Output and platform directories are based on bootstrap
        self.output_dir = os.path.join(self.bootstrap_dir, 'output')
        self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
        self.bt_dir = os.path.join(self.platform_dir, 'bt')
        self.sysroot = self.args.sysroot
        self.libdir = self.args.libdir
        self.install_dir = os.path.join(self.output_dir, 'install')

        assert os.path.samefile(self.bt_dir,
                                os.path.dirname(__file__)), "Please rerun bootstrap for the current project!"

        # If default target isn't set, build everything
        self.target = 'all'
        if hasattr(self.args, 'target') and self.args.target:
            self.target = self.args.target

        target_use = self.args.use if self.args.use else []

        # Unless set, always build test code
        if not self.args.notest:
            target_use.append('test')

        self.use = UseFlags(target_use)

        # Validate platform directory
        assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
        assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'

        # Make sure output directory exists (or create it)
        os.makedirs(self.output_dir, exist_ok=True)

        # Set some default attributes
        self.libbase_ver = None

        self.configure_environ()

    def _generate_rustflags(self):
        """ Rustflags to include for the build.
      """
        rust_flags = [
            '-L',
            '{}/out/Default'.format(self.output_dir),
            '-C',
            'link-arg=-Wl,--allow-multiple-definition',
            # exclude uninteresting warnings
            '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
            '-Cstrip=debuginfo',
            '-Copt-level=z',
        ]

        return ' '.join(rust_flags)

    def configure_environ(self):
        """ Configure environment variables for GN and Cargo.
        """
        self.env = os.environ.copy()

        # Make sure cargo home dir exists and has a bin directory
        cargo_home = os.path.join(self.output_dir, 'cargo_home')
        os.makedirs(cargo_home, exist_ok=True)
        os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)

        # Configure Rust env variables
        self.custom_env = {}
        self.custom_env['CARGO_TARGET_DIR'] = self.output_dir
        self.custom_env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
        self.custom_env['RUSTFLAGS'] = self._generate_rustflags()
        self.custom_env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
        self.custom_env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
        self.custom_env['CXX_OUTDIR'] = self._gn_default_output()

        # On ChromeOS, this is /usr/bin/grpc_rust_plugin
        # In the container, this is /root/.cargo/bin/grpc_rust_plugin
        self.custom_env['GRPC_RUST_PLUGIN_PATH'] = shutil.which('grpc_rust_plugin')
        self.env.update(self.custom_env)

    def print_env(self):
        """ Print the custom environment variables that are used in build.

        Useful so that external tools can mimic the environment to be the same
        as build.py, e.g. rust-analyzer.
        """
        for k, v in self.custom_env.items():
            print("export {}='{}'".format(k, v))

    def run_command(self, target, args, cwd=None, env=None):
        """ Run command and stream the output.
        """
        # Set some defaults
        if not cwd:
            cwd = self.platform_dir
        if not env:
            env = self.env

        for k, v in env.items():
            if env[k] is None:
                env[k] = ""

        log_file = os.path.join(self.output_dir, '{}.log'.format(target))
        with open(log_file, 'wb') as lf:
            rc = 0
            process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
            while True:
                line = process.stdout.readline()
                print(line.decode('utf-8'), end="")
                lf.write(line)
                if not line:
                    rc = process.poll()
                    if rc is not None:
                        break

                    time.sleep(0.1)

            if rc != 0:
                raise Exception("Return code is {}".format(rc))

    def _get_basever(self):
        if self.libbase_ver:
            return self.libbase_ver

        self.libbase_ver = os.environ.get('BASE_VER', '')
        if not self.libbase_ver:
            base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
            try:
                with open(base_file, 'r') as f:
                    self.libbase_ver = f.read().strip('\n')
            except:
                self.libbase_ver = 'NOT-INSTALLED'

        return self.libbase_ver

    def _gn_default_output(self):
        return os.path.join(self.output_dir, 'out/Default')

    def _gn_configure(self):
        """ Configure all required parameters for platform2.

        Mostly copied from //common-mk/platform2.py
        """
        clang = not self.args.no_clang

        def to_gn_string(s):
            return '"%s"' % s.replace('"', '\\"')

        def to_gn_list(strs):
            return '[%s]' % ','.join([to_gn_string(s) for s in strs])

        def to_gn_args_args(gn_args):
            for k, v in gn_args.items():
                if isinstance(v, bool):
                    v = str(v).lower()
                elif isinstance(v, list):
                    v = to_gn_list(v)
                elif isinstance(v, six.string_types):
                    v = to_gn_string(v)
                else:
                    raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
                yield '%s=%s' % (k.replace('-', '_'), v)

        gn_args = {
            'platform_subdir': 'bt',
            'cc': 'clang' if clang else 'gcc',
            'cxx': 'clang++' if clang else 'g++',
            'ar': 'llvm-ar' if clang else 'ar',
            'pkg-config': 'pkg-config',
            'clang_cc': clang,
            'clang_cxx': clang,
            'OS': 'linux',
            'sysroot': self.sysroot,
            'libdir': os.path.join(self.sysroot, self.libdir),
            'build_root': self.output_dir,
            'platform2_root': self.platform_dir,
            'libbase_ver': self._get_basever(),
            'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
            'external_cflags': [],
            'external_cxxflags': ["-DNDEBUG"],
            'enable_werror': True,
        }

        if clang:
            # Make sure to mark the clang use flag as true
            self.use.set_flag('clang', True)
            gn_args['external_cxxflags'] += ['-I/usr/include/']

        gn_args_args = list(to_gn_args_args(gn_args))
        use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
        gn_args_args += ['use={%s}' % (' '.join(use_args))]

        gn_args = [
            'gn',
            'gen',
        ]

        if self.args.verbose:
            gn_args.append('-v')

        gn_args += [
            '--root=%s' % self.platform_dir,
            '--args=%s' % ' '.join(gn_args_args),
            self._gn_default_output(),
        ]

        if 'PKG_CONFIG_PATH' in self.env:
            print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])

        self.run_command('configure', gn_args)

    def _gn_build(self, target):
        """ Generate the ninja command for the target and run it.
        """
        args = ['%s:%s' % ('bt', target)]
        ninja_args = ['ninja', '-C', self._gn_default_output()]
        if self.jobs:
            ninja_args += ['-j', str(self.jobs)]
        ninja_args += args

        if self.args.verbose:
            ninja_args.append('-v')

        self.run_command('build', ninja_args)

    def _rust_configure(self):
        """ Generate config file at cargo_home so we use vendored crates.
        """
        template = """
        [source.systembt]
        directory = "{}/external/rust/vendor"

        [source.crates-io]
        replace-with = "systembt"
        local-registry = "/nonexistent"
        """

        if not self.args.no_vendored_rust:
            contents = template.format(self.platform_dir)
            with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
                f.write(contents)

    def _rust_build(self):
        """ Run `cargo build` from platform2/bt directory.
        """
        cmd = ['cargo', 'build']
        if not self.args.rust_debug:
            cmd.append('--release')

        self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)

    def _target_prepare(self):
        """ Target to prepare the output directory for building.

        This runs gn gen to generate all rquired files and set up the Rust
        config properly. This will be run
        """
        self._gn_configure()
        self._rust_configure()

    def _target_hosttools(self):
        """ Build the tools target in an already prepared environment.
        """
        self._gn_build('tools')

        # Also copy bluetooth_packetgen to CARGO_HOME so it's available
        shutil.copy(os.path.join(self._gn_default_output(), 'bluetooth_packetgen'),
                    os.path.join(self.env['CARGO_HOME'], 'bin'))

    def _target_docs(self):
        """Build the Rust docs."""
        self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)

    def _target_rust(self):
        """ Build rust artifacts in an already prepared environment.
        """
        self._rust_build()

    def _target_main(self):
        """ Build the main GN artifacts in an already prepared environment.
        """
        self._gn_build('all')

    def _target_test(self):
        """ Runs the host tests.
        """
        # Rust tests first
        rust_test_cmd = ['cargo', 'test']
        if not self.args.rust_debug:
            rust_test_cmd.append('--release')

        if self.args.test_name:
            rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"]

        self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)

        # Host tests second based on host test list
        for t in HOST_TESTS:
            self.run_command('test', [os.path.join(self.output_dir, 'out/Default', t)],
                             cwd=os.path.join(self.output_dir),
                             env=self.env)

    def _target_clippy(self):
        """ Runs cargo clippy, a collection of lints to catch common mistakes.
        """
        cmd = ['cargo', 'clippy']
        self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)

    def _target_utils(self):
        """ Builds the utility applications.
        """
        rust_targets = ['hcidoc']

        # Build targets
        for target in rust_targets:
            self.run_command('utils', ['cargo', 'build', '-p', target],
                             cwd=os.path.join(self.platform_dir, 'bt'),
                             env=self.env)

    def _target_install(self):
        """ Installs files required to run Floss to install directory.
        """
        # First make the install directory
        prefix = self.install_dir
        os.makedirs(prefix, exist_ok=True)

        # Next save the cwd and change to install directory
        last_cwd = os.getcwd()
        os.chdir(prefix)

        bindir = os.path.join(self.output_dir, 'debug')
        srcdir = os.path.dirname(__file__)

        install_map = [
            {
                'src': os.path.join(bindir, 'btadapterd'),
                'dst': 'usr/libexec/bluetooth/btadapterd',
                'strip': True
            },
            {
                'src': os.path.join(bindir, 'btmanagerd'),
                'dst': 'usr/libexec/bluetooth/btmanagerd',
                'strip': True
            },
            {
                'src': os.path.join(bindir, 'btclient'),
                'dst': 'usr/local/bin/btclient',
                'strip': True
            },
        ]

        for v in install_map:
            src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
            dst = os.path.join(prefix, partial_dst)

            # Create dst directory first and copy file there
            os.makedirs(os.path.dirname(dst), exist_ok=True)
            print('Installing {}'.format(dst))
            shutil.copy(src, dst)

            # Binary should be marked for strip and no-strip option shouldn't be
            # set. No-strip is useful while debugging.
            if strip and not self.args.no_strip:
                self.run_command('install', ['llvm-strip', dst])

        # Put all files into a tar.gz for easier installation
        tar_location = os.path.join(prefix, 'floss.tar.gz')
        with tarfile.open(tar_location, 'w:gz') as tar:
            for v in install_map:
                tar.add(v['dst'])

        print('Tarball created at {}'.format(tar_location))

    def _target_bloat(self):
        """Run cargo bloat on workspace.
        """
        crate_paths = [
            os.path.join(self.platform_dir, 'bt', 'system', 'gd', 'rust', 'linux', 'mgmt'),
            os.path.join(self.platform_dir, 'bt', 'system', 'gd', 'rust', 'linux', 'service'),
            os.path.join(self.platform_dir, 'bt', 'system', 'gd', 'rust', 'linux', 'client')
        ]
        for crate in crate_paths:
            self.run_command('bloat', ['cargo', 'bloat', '--release', '--crates', '--wide'], cwd=crate, env=self.env)

    def _target_clean(self):
        """ Delete the output directory entirely.
        """
        shutil.rmtree(self.output_dir)

        # Remove Cargo.lock that may have become generated
        cargo_lock_files = [
            os.path.join(self.platform_dir, 'bt', 'Cargo.lock'),
        ]
        for lock_file in cargo_lock_files:
            try:
                os.remove(lock_file)
                print('Removed {}'.format(lock_file))
            except FileNotFoundError:
                pass

    def _target_all(self):
        """ Build all common targets (skipping doc, test, and clean).
        """
        self._target_prepare()
        self._target_hosttools()
        self._target_main()
        self._target_rust()

    def build(self):
        """ Builds according to self.target
        """
        print('Building target ', self.target)

        # Validate that the target is valid
        if self.target not in VALID_TARGETS:
            print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS))
            return

        if self.target == 'prepare':
            self._target_prepare()
        elif self.target == 'hosttools':
            self._target_hosttools()
        elif self.target == 'rust':
            self._target_rust()
        elif self.target == 'docs':
            self._target_docs()
        elif self.target == 'main':
            self._target_main()
        elif self.target == 'test':
            self._target_test()
        elif self.target == 'clippy':
            self._target_clippy()
        elif self.target == 'clean':
            self._target_clean()
        elif self.target == 'install':
            self._target_install()
        elif self.target == 'utils':
            self._target_utils()
        elif self.target == 'bloat':
            self._target_bloat()
        elif self.target == 'all':
            self._target_all()


# Default to 10 min timeouts on all git operations.
GIT_TIMEOUT_SEC = 600


class Bootstrap():

    def __init__(self, base_dir, bt_dir, partial_staging, clone_timeout):
        """ Construct bootstrapper.

        Args:
            base_dir: Where to stage everything.
            bt_dir: Where bluetooth source is kept (will be symlinked)
            partial_staging: Whether to do a partial clone for staging.
            clone_timeout: Timeout for clone operations.
        """
        self.base_dir = os.path.abspath(base_dir)
        self.bt_dir = os.path.abspath(bt_dir)
        self.partial_staging = partial_staging
        self.clone_timeout = clone_timeout

        # Create base directory if it doesn't already exist
        os.makedirs(self.base_dir, exist_ok=True)

        if not os.path.isdir(self.bt_dir):
            raise Exception('{} is not a valid directory'.format(self.bt_dir))

        self.git_dir = os.path.join(self.base_dir, 'repos')
        self.staging_dir = os.path.join(self.base_dir, 'staging')
        self.output_dir = os.path.join(self.base_dir, 'output')
        self.external_dir = os.path.join(self.base_dir, 'staging', 'external')

        self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')

    def _run_with_timeout(self, cmd, cwd, timeout=None):
        """Runs a command using subprocess.check_output. """
        print('Running command: {} [at cwd={}]'.format(' '.join(cmd), cwd))
        with subprocess.Popen(cmd, cwd=cwd) as proc:
            try:
                outs, errs = proc.communicate(timeout=timeout)
            except subprocess.TimeoutExpired:
                proc.kill()
                outs, errs = proc.communicate()
                print('Timeout on {}'.format(' '.join(cmd)), file=sys.stderr)
                raise

            if proc.returncode != 0:
                raise Exception('Cmd {} had return code {}'.format(' '.join(cmd), proc.returncode))

    def _update_platform2(self):
        """Updates repositories used for build."""
        for project in BOOTSTRAP_GIT_REPOS.keys():
            cwd = os.path.join(self.git_dir, project)
            (repo, commit) = BOOTSTRAP_GIT_REPOS[project]

            # Update to required commit when necessary or pull the latest code.
            if commit is not None:
                head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=cwd).strip()
                if head != commit:
                    subprocess.check_call(['git', 'fetch'], cwd=cwd)
                    subprocess.check_call(['git', 'checkout', commit], cwd=cwd)
            else:
                subprocess.check_call(['git', 'pull'], cwd=cwd)

    def _setup_platform2(self):
        """ Set up platform2.

        This will check out all the git repos and symlink everything correctly.
        """

        # Create all directories we will need to use
        for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
            os.makedirs(dirpath, exist_ok=True)

        # If already set up, only update platform2
        if os.path.isfile(self.dir_setup_complete):
            print('{} already set-up. Updating instead.'.format(self.base_dir))
            self._update_platform2()
        else:
            clone_options = []
            # When doing a partial staging, we use a treeless clone which allows
            # us to access all commits but downloads things on demand. This
            # helps speed up the initial git clone during builds but isn't good
            # for long-term development.
            if self.partial_staging:
                clone_options = ['--filter=tree:0']
            # Check out all repos in git directory
            for project in BOOTSTRAP_GIT_REPOS.keys():
                (repo, commit) = BOOTSTRAP_GIT_REPOS[project]

                # Try repo clone several times.
                # Currently, we set timeout on this operation after
                # |self.clone_timeout|. If it fails, try to recover.
                tries = 2
                for x in range(tries):
                    try:
                        self._run_with_timeout(['git', 'clone', repo, project] + clone_options,
                                               cwd=self.git_dir,
                                               timeout=self.clone_timeout)
                    except subprocess.TimeoutExpired:
                        shutil.rmtree(os.path.join(self.git_dir, project))
                        if x == tries - 1:
                            raise
                    # All other exceptions should raise
                    except:
                        raise
                    # No exceptions/problems should not retry.
                    else:
                        break

                # Pin to commit.
                if commit is not None:
                    subprocess.check_call(['git', 'checkout', commit], cwd=os.path.join(self.git_dir, project))

        # Symlink things
        symlinks = [
            (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
            (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
            (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
            (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
            (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
            (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
        ]

        # Create symlinks
        for pairs in symlinks:
            (src, dst) = pairs
            try:
                os.unlink(dst)
            except Exception as e:
                print(e)
            os.symlink(src, dst)

        # Write to setup complete file so we don't repeat this step
        with open(self.dir_setup_complete, 'w') as f:
            f.write('Setup complete.')

    def _pretty_print_install(self, install_cmd, packages, line_limit=80):
        """ Pretty print an install command.

        Args:
            install_cmd: Prefixed install command.
            packages: Enumerate packages and append them to install command.
            line_limit: Number of characters per line.

        Return:
            Array of lines to join and print.
        """
        install = [install_cmd]
        line = '  '
        # Remainder needed = space + len(pkg) + space + \
        # Assuming 80 character lines, that's 80 - 3 = 77
        line_limit = line_limit - 3
        for pkg in packages:
            if len(line) + len(pkg) < line_limit:
                line = '{}{} '.format(line, pkg)
            else:
                install.append(line)
                line = '  {} '.format(pkg)

        if len(line) > 0:
            install.append(line)

        return install

    def _check_package_installed(self, package, cmd, predicate):
        """Check that the given package is installed.

        Args:
            package: Check that this package is installed.
            cmd: Command prefix to check if installed (package appended to end)
            predicate: Function/lambda to check if package is installed based
                       on output. Takes string output and returns boolean.

        Return:
            True if package is installed.
        """
        try:
            output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
            is_installed = predicate(output.decode('utf-8'))
            print('  {} is {}'.format(package, 'installed' if is_installed else 'missing'))

            return is_installed
        except Exception as e:
            print(e)
            return False

    def _get_command_output(self, cmd):
        """Runs the command and gets the output.

        Args:
            cmd: Command to run.

        Return:
            Tuple (Success, Output). Success represents if the command ran ok.
        """
        try:
            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            return (True, output.decode('utf-8').split('\n'))
        except Exception as e:
            print(e)
            return (False, "")

    def _print_missing_packages(self):
        """Print any missing packages found via apt.

        This will find any missing packages necessary for build using apt and
        print it out as an apt-get install printf.
        """
        print('Checking for any missing packages...')

        (success, output) = self._get_command_output(APT_PKG_LIST)
        if not success:
            raise Exception("Could not query apt for packages.")

        packages_installed = {}
        for line in output:
            if 'installed' in line:
                split = line.split('/', 2)
                packages_installed[split[0]] = True

        need_packages = []
        for pkg in REQUIRED_APT_PACKAGES:
            if pkg not in packages_installed:
                need_packages.append(pkg)

        # No packages need to be installed
        if len(need_packages) == 0:
            print('+ All required packages are installed')
            return

        install = self._pretty_print_install('sudo apt-get install', need_packages)

        # Print all lines so they can be run in cmdline
        print('Missing system packages. Run the following command: ')
        print(' \\\n'.join(install))

    def _print_missing_rust_packages(self):
        """Print any missing packages found via cargo.

        This will find any missing packages necessary for build using cargo and
        print it out as a cargo-install printf.
        """
        print('Checking for any missing cargo packages...')

        (success, output) = self._get_command_output(CARGO_PKG_LIST)
        if not success:
            raise Exception("Could not query cargo for packages.")

        packages_installed = {}
        for line in output:
            # Cargo installed packages have this format
            # <crate name> <version>:
            #   <binary name>
            # We only care about the crates themselves
            if ':' not in line:
                continue

            split = line.split(' ', 2)
            packages_installed[split[0]] = True

        need_packages = []
        for pkg in REQUIRED_CARGO_PACKAGES:
            if pkg not in packages_installed:
                need_packages.append(pkg)

        # No packages to be installed
        if len(need_packages) == 0:
            print('+ All required cargo packages are installed')
            return

        install = self._pretty_print_install('cargo install', need_packages)
        print('Missing cargo packages. Run the following command: ')
        print(' \\\n'.join(install))

    def bootstrap(self):
        """ Bootstrap the Linux build."""
        self._setup_platform2()
        self._print_missing_packages()
        self._print_missing_rust_packages()


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Simple build for host.')
    parser.add_argument('--bootstrap-dir',
                        help='Directory to run bootstrap on (or was previously run on).',
                        default="~/.floss")
    parser.add_argument('--run-bootstrap',
                        help='Run bootstrap code to verify build env is ok to build.',
                        default=False,
                        action='store_true')
    parser.add_argument('--print-env',
                        help='Print environment variables used for build.',
                        default=False,
                        action='store_true')
    parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
    parser.add_argument('--no-strip',
                        help='Skip stripping binaries during install.',
                        default=False,
                        action='store_true')
    parser.add_argument('--use', help='Set a specific use flag.')
    parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
    parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
    parser.add_argument('--target', help='Run specific build target')
    parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
    parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
    parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
    parser.add_argument('--no-vendored-rust',
                        help='Do not use vendored rust crates',
                        default=False,
                        action='store_true')
    parser.add_argument('--verbose', help='Verbose logs for build.')
    parser.add_argument('--rust-debug', help='Build Rust code as debug.', default=False, action='store_true')
    parser.add_argument(
        '--partial-staging',
        help='Bootstrap git repositories with partial clones. Use to speed up initial git clone for automated builds.',
        default=False,
        action='store_true')
    parser.add_argument('--clone-timeout',
                        help='Timeout for repository cloning during bootstrap.',
                        default=GIT_TIMEOUT_SEC,
                        type=int)
    args = parser.parse_args()

    # Make sure we get absolute path + expanded path for bootstrap directory
    args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))

    # Possible values for machine() come from 'uname -m'
    # Since this script only runs on Linux, x86_64 machines must have this value
    if platform.machine() != 'x86_64':
        raise Exception("Only x86_64 machines are currently supported by this build script.")

    if args.run_bootstrap:
        bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__), args.partial_staging, args.clone_timeout)
        bootstrap.bootstrap()
    elif args.print_env:
        build = HostBuild(args)
        build.print_env()
    else:
        build = HostBuild(args)
        build.build()