X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=tools%2Fbuildman%2Fbuilder.py;h=a5a2ffdfdf2c76d86fb78d765d9977245da652ac;hb=8da19df5b504f1cbe79e13f859bc229c042aded5;hp=48408ff0b14ec8ee950df9ec396721f5d6929a60;hpb=dcdb61a084bd3840d0987a5d01ae506c11efca1b;p=oweals%2Fu-boot.git diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 48408ff0b1..a5a2ffdfdf 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -1,26 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0+ # Copyright (c) 2013 The Chromium OS Authors. # # Bloat-o-meter code used here Copyright 2004 Matt Mackall # -# SPDX-License-Identifier: GPL-2.0+ -# import collections -import errno from datetime import datetime, timedelta import glob import os import re import Queue import shutil +import signal import string import sys import threading import time +import builderthread import command import gitutil import terminal +from terminal import Print import toolchain @@ -93,418 +94,52 @@ u-boot/ source directory # Possible build outcomes OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) -# Translate a commit subject into a valid filename -trans_valid_chars = string.maketrans("/: ", "---") - - -def Mkdir(dirname): - """Make a directory if it doesn't already exist. - - Args: - dirname: Directory to create - """ - try: - os.mkdir(dirname) - except OSError as err: - if err.errno == errno.EEXIST: - pass - else: - raise - -class BuilderJob: - """Holds information about a job to be performed by a thread - - Members: - board: Board object to build - commits: List of commit options to build. - """ - def __init__(self): - self.board = None - self.commits = [] - - -class ResultThread(threading.Thread): - """This thread processes results from builder threads. - - It simply passes the results on to the builder. There is only one - result thread, and this helps to serialise the build output. - """ - def __init__(self, builder): - """Set up a new result thread - - Args: - builder: Builder which will be sent each result - """ - threading.Thread.__init__(self) - self.builder = builder - - def run(self): - """Called to start up the result thread. - - We collect the next result job and pass it on to the build. - """ - while True: - result = self.builder.out_queue.get() - self.builder.ProcessResult(result) - self.builder.out_queue.task_done() - - -class BuilderThread(threading.Thread): - """This thread builds U-Boot for a particular board. - - An input queue provides each new job. We run 'make' to build U-Boot - and then pass the results on to the output queue. - - Members: - builder: The builder which contains information we might need - thread_num: Our thread number (0-n-1), used to decide on a - temporary directory - """ - def __init__(self, builder, thread_num): - """Set up a new builder thread""" - threading.Thread.__init__(self) - self.builder = builder - self.thread_num = thread_num - - def Make(self, commit, brd, stage, cwd, *args, **kwargs): - """Run 'make' on a particular commit and board. - - The source code will already be checked out, so the 'commit' - argument is only for information. - - Args: - commit: Commit object that is being built - brd: Board object that is being built - stage: Stage of the build. Valid stages are: - distclean - can be called to clean source - config - called to configure for a board - build - the main make invocation - it does the build - args: A list of arguments to pass to 'make' - kwargs: A list of keyword arguments to pass to command.RunPipe() - - Returns: - CommandResult object - """ - return self.builder.do_make(commit, brd, stage, cwd, *args, - **kwargs) - - def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, - force_build_failures): - """Build a particular commit. - - If the build is already done, and we are not forcing a build, we skip - the build and just return the previously-saved results. - - Args: - commit_upto: Commit number to build (0...n-1) - brd: Board object to build - work_dir: Directory to which the source will be checked out - do_config: True to run a make _defconfig on the source - force_build: Force a build even if one was previously done - force_build_failures: Force a bulid if the previous result showed - failure - - Returns: - tuple containing: - - CommandResult object containing the results of the build - - boolean indicating whether 'make config' is still needed - """ - # Create a default result - it will be overwritte by the call to - # self.Make() below, in the event that we do a build. - result = command.CommandResult() - result.return_code = 0 - if self.builder.in_tree: - out_dir = work_dir - else: - out_dir = os.path.join(work_dir, 'build') - - # Check if the job was already completed last time - done_file = self.builder.GetDoneFile(commit_upto, brd.target) - result.already_done = os.path.exists(done_file) - will_build = (force_build or force_build_failures or - not result.already_done) - if result.already_done and will_build: - # Get the return code from that build and use it - with open(done_file, 'r') as fd: - result.return_code = int(fd.readline()) - err_file = self.builder.GetErrFile(commit_upto, brd.target) - if os.path.exists(err_file) and os.stat(err_file).st_size: - result.stderr = 'bad' - elif not force_build: - # The build passed, so no need to build it again - will_build = False - - if will_build: - # We are going to have to build it. First, get a toolchain - if not self.toolchain: - try: - self.toolchain = self.builder.toolchains.Select(brd.arch) - except ValueError as err: - result.return_code = 10 - result.stdout = '' - result.stderr = str(err) - # TODO(sjg@chromium.org): This gets swallowed, but needs - # to be reported. - - if self.toolchain: - # Checkout the right commit - if commit_upto is not None: - commit = self.builder.commits[commit_upto] - if self.builder.checkout: - git_dir = os.path.join(work_dir, '.git') - gitutil.Checkout(commit.hash, git_dir, work_dir, - force=True) - else: - commit = self.builder.commit # Ick, fix this for BuildCommits() - - # Set up the environment and command line - env = self.toolchain.MakeEnvironment() - Mkdir(out_dir) - args = [] - if not self.builder.in_tree: - args.append('O=build') - args.append('-s') - if self.builder.num_jobs is not None: - args.extend(['-j', str(self.builder.num_jobs)]) - config_args = ['%s_defconfig' % brd.target] - config_out = '' - args.extend(self.builder.toolchains.GetMakeArguments(brd)) - - # If we need to reconfigure, do that now - if do_config: - result = self.Make(commit, brd, 'distclean', work_dir, - 'distclean', *args, env=env) - result = self.Make(commit, brd, 'config', work_dir, - *(args + config_args), env=env) - config_out = result.combined - do_config = False # No need to configure next time - if result.return_code == 0: - result = self.Make(commit, brd, 'build', work_dir, *args, - env=env) - result.stdout = config_out + result.stdout - else: - result.return_code = 1 - result.stderr = 'No tool chain for %s\n' % brd.arch - result.already_done = False - - result.toolchain = self.toolchain - result.brd = brd - result.commit_upto = commit_upto - result.out_dir = out_dir - return result, do_config - - def _WriteResult(self, result, keep_outputs): - """Write a built result to the output directory. - - Args: - result: CommandResult object containing result to write - keep_outputs: True to store the output binaries, False - to delete them - """ - # Fatal error - if result.return_code < 0: - return - - # Aborted? - if result.stderr and 'No child processes' in result.stderr: - return - - if result.already_done: - return - - # Write the output and stderr - output_dir = self.builder._GetOutputDir(result.commit_upto) - Mkdir(output_dir) - build_dir = self.builder.GetBuildDir(result.commit_upto, - result.brd.target) - Mkdir(build_dir) - - outfile = os.path.join(build_dir, 'log') - with open(outfile, 'w') as fd: - if result.stdout: - fd.write(result.stdout) - - errfile = self.builder.GetErrFile(result.commit_upto, - result.brd.target) - if result.stderr: - with open(errfile, 'w') as fd: - fd.write(result.stderr) - elif os.path.exists(errfile): - os.remove(errfile) - - if result.toolchain: - # Write the build result and toolchain information. - done_file = self.builder.GetDoneFile(result.commit_upto, - result.brd.target) - with open(done_file, 'w') as fd: - fd.write('%s' % result.return_code) - with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: - print >>fd, 'gcc', result.toolchain.gcc - print >>fd, 'path', result.toolchain.path - print >>fd, 'cross', result.toolchain.cross - print >>fd, 'arch', result.toolchain.arch - fd.write('%s' % result.return_code) - - with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: - print >>fd, 'gcc', result.toolchain.gcc - print >>fd, 'path', result.toolchain.path - - # Write out the image and function size information and an objdump - env = result.toolchain.MakeEnvironment() - lines = [] - for fname in ['u-boot', 'spl/u-boot-spl']: - cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] - nm_result = command.RunPipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) - if nm_result.stdout: - nm = self.builder.GetFuncSizesFile(result.commit_upto, - result.brd.target, fname) - with open(nm, 'w') as fd: - print >>fd, nm_result.stdout, - - cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] - dump_result = command.RunPipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) - rodata_size = '' - if dump_result.stdout: - objdump = self.builder.GetObjdumpFile(result.commit_upto, - result.brd.target, fname) - with open(objdump, 'w') as fd: - print >>fd, dump_result.stdout, - for line in dump_result.stdout.splitlines(): - fields = line.split() - if len(fields) > 5 and fields[1] == '.rodata': - rodata_size = fields[2] - - cmd = ['%ssize' % self.toolchain.cross, fname] - size_result = command.RunPipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) - if size_result.stdout: - lines.append(size_result.stdout.splitlines()[1] + ' ' + - rodata_size) - - # Write out the image sizes file. This is similar to the output - # of binutil's 'size' utility, but it omits the header line and - # adds an additional hex value at the end of each line for the - # rodata size - if len(lines): - sizes = self.builder.GetSizesFile(result.commit_upto, - result.brd.target) - with open(sizes, 'w') as fd: - print >>fd, '\n'.join(lines) - - # Now write the actual build output - if keep_outputs: - patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', - 'include/autoconf.mk', 'spl/u-boot-spl', - 'spl/u-boot-spl.bin'] - for pattern in patterns: - file_list = glob.glob(os.path.join(result.out_dir, pattern)) - for fname in file_list: - shutil.copy(fname, build_dir) - - - def RunJob(self, job): - """Run a single job - - A job consists of a building a list of commits for a particular board. - - Args: - job: Job to build - """ - brd = job.board - work_dir = self.builder.GetThreadDir(self.thread_num) - self.toolchain = None - if job.commits: - # Run 'make board_defconfig' on the first commit - do_config = True - commit_upto = 0 - force_build = False - for commit_upto in range(0, len(job.commits), job.step): - result, request_config = self.RunCommit(commit_upto, brd, - work_dir, do_config, - force_build or self.builder.force_build, - self.builder.force_build_failures) - failed = result.return_code or result.stderr - did_config = do_config - if failed and not do_config: - # If our incremental build failed, try building again - # with a reconfig. - if self.builder.force_config_on_failure: - result, request_config = self.RunCommit(commit_upto, - brd, work_dir, True, True, False) - did_config = True - if not self.builder.force_reconfig: - do_config = request_config - - # If we built that commit, then config is done. But if we got - # an warning, reconfig next time to force it to build the same - # files that created warnings this time. Otherwise an - # incremental build may not build the same file, and we will - # think that the warning has gone away. - # We could avoid this by using -Werror everywhere... - # For errors, the problem doesn't happen, since presumably - # the build stopped and didn't generate output, so will retry - # that file next time. So we could detect warnings and deal - # with them specially here. For now, we just reconfigure if - # anything goes work. - # Of course this is substantially slower if there are build - # errors/warnings (e.g. 2-3x slower even if only 10% of builds - # have problems). - if (failed and not result.already_done and not did_config and - self.builder.force_config_on_failure): - # If this build failed, try the next one with a - # reconfigure. - # Sometimes if the board_config.h file changes it can mess - # with dependencies, and we get: - # make: *** No rule to make target `include/autoconf.mk', - # needed by `depend'. - do_config = True - force_build = True - else: - force_build = False - if self.builder.force_config_on_failure: - if failed: - do_config = True - result.commit_upto = commit_upto - if result.return_code < 0: - raise ValueError('Interrupt') - - # We have the build results, so output the result - self._WriteResult(result, job.keep_outputs) - self.builder.out_queue.put(result) - else: - # Just build the currently checked-out build - result = self.RunCommit(None, True) - result.commit_upto = self.builder.upto - self.builder.out_queue.put(result) - - def run(self): - """Our thread's run function - - This thread picks a job from the queue, runs it, and then goes to the - next job. - """ - alive = True - while True: - job = self.builder.queue.get() - try: - if self.builder.active and alive: - self.RunJob(job) - except Exception as err: - alive = False - print err - self.builder.queue.task_done() - +# Translate a commit subject into a valid filename (and handle unicode) +trans_valid_chars = string.maketrans('/: ', '---') +trans_valid_chars = trans_valid_chars.decode('latin-1') + +BASE_CONFIG_FILENAMES = [ + 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' +] + +EXTRA_CONFIG_FILENAMES = [ + '.config', '.config-spl', '.config-tpl', + 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', + 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', +] + +class Config: + """Holds information about configuration settings for a board.""" + def __init__(self, config_filename, target): + self.target = target + self.config = {} + for fname in config_filename: + self.config[fname] = {} + + def Add(self, fname, key, value): + self.config[fname][key] = value + + def __hash__(self): + val = 0 + for fname in self.config: + for key, value in self.config[fname].iteritems(): + print key, value + val = val ^ hash(key) & hash(value) + return val + +class Environment: + """Holds information about environment variables for a board.""" + def __init__(self, target): + self.target = target + self.environment = {} + + def Add(self, key, value): + self.environment[key] = value class Builder: """Class for building U-Boot for a particular commit. Public members: (many should ->private) - active: True if the builder is active and has not been stopped already_done: Number of builds already completed base_dir: Base directory to use for builder checkout: True to check out source, False to skip that step. @@ -544,6 +179,7 @@ class Builder: Private members: _base_board_dict: Last-summarised Dict of boards _base_err_lines: Last-summarised list of errors + _base_warn_lines: Last-summarised list of warnings _build_period_us: Time taken for a single build (float object). _complete_delay: Expected delay until completion (timedelta) _next_delay_update: Next time we plan to display a progress update @@ -568,15 +204,28 @@ class Builder: value is itself a dictionary: key: function name value: Size of function in bytes + config: Dictionary keyed by filename - e.g. '.config'. Each + value is itself a dictionary: + key: config name + value: config value + environment: Dictionary keyed by environment variable, Each + value is the value of environment variable. """ - def __init__(self, rc, err_lines, sizes, func_sizes): + def __init__(self, rc, err_lines, sizes, func_sizes, config, + environment): self.rc = rc self.err_lines = err_lines self.sizes = sizes self.func_sizes = func_sizes + self.config = config + self.environment = environment def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, - gnu_make='make', checkout=True, show_unknown=True, step=1): + gnu_make='make', checkout=True, show_unknown=True, step=1, + no_subdirs=False, full_path=False, verbose_build=False, + incremental=False, per_board_out_dir=False, + config_only=False, squash_config_y=False, + warnings_as_errors=False): """Create a new Builder object Args: @@ -590,12 +239,23 @@ class Builder: This is used for testing. show_unknown: Show unknown boards (those not built) in summary step: 1 to process every commit, n to process every nth commit + no_subdirs: Don't create subdirectories when building current + source for a single board + full_path: Return the full path in CROSS_COMPILE and don't set + PATH + verbose_build: Run build with V=1 and don't use 'make -s' + incremental: Always perform incremental builds; don't run make + mrproper when configuring + per_board_out_dir: Build in a separate persistent directory per + board rather than a thread-specific directory + config_only: Only configure each build, don't build it + squash_config_y: Convert CONFIG options with the value 'y' to '1' + warnings_as_errors: Treat all compiler warnings as errors """ self.toolchains = toolchains self.base_dir = base_dir self._working_dir = os.path.join(base_dir, '.bm-work') self.threads = [] - self.active = True self.do_make = self.Make self.gnu_make = gnu_make self.checkout = checkout @@ -614,19 +274,35 @@ class Builder: self.force_reconfig = False self._step = step self.in_tree = False - + self._error_lines = 0 + self.no_subdirs = no_subdirs + self.full_path = full_path + self.verbose_build = verbose_build + self.config_only = config_only + self.squash_config_y = squash_config_y + self.config_filenames = BASE_CONFIG_FILENAMES + if not self.squash_config_y: + self.config_filenames += EXTRA_CONFIG_FILENAMES + + self.warnings_as_errors = warnings_as_errors self.col = terminal.Color() + self._re_function = re.compile('(.*): In function.*') + self._re_files = re.compile('In file included from.*') + self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') + self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') + self.queue = Queue.Queue() self.out_queue = Queue.Queue() for i in range(self.num_threads): - t = BuilderThread(self, i) + t = builderthread.BuilderThread(self, i, incremental, + per_board_out_dir) t.setDaemon(True) t.start() self.threads.append(t) self.last_line_len = 0 - t = ResultThread(self) + t = builderthread.ResultThread(self) t.setDaemon(True) t.start() self.threads.append(t) @@ -634,11 +310,39 @@ class Builder: ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] self.re_make_err = re.compile('|'.join(ignore_lines)) + # Handle existing graceful with SIGINT / Ctrl-C + signal.signal(signal.SIGINT, self.signal_handler) + def __del__(self): """Get rid of all threads created by the builder""" for t in self.threads: del t + def signal_handler(self, signal, frame): + sys.exit(1) + + def SetDisplayOptions(self, show_errors=False, show_sizes=False, + show_detail=False, show_bloat=False, + list_error_boards=False, show_config=False, + show_environment=False): + """Setup display options for the builder. + + show_errors: True to show summarised error/warning info + show_sizes: Show size deltas + show_detail: Show detail for each board + show_bloat: Show detail for each function + list_error_boards: Show the boards which caused each error/warning + show_config: Show config deltas + show_environment: Show environment deltas + """ + self._show_errors = show_errors + self._show_sizes = show_sizes + self._show_detail = show_detail + self._show_bloat = show_bloat + self._list_error_boards = list_error_boards + self._show_config = show_config + self._show_environment = show_environment + def _AddTimestamp(self): """Add a new timestamp to the list and record the build period. @@ -679,8 +383,8 @@ class Builder: length: Length of new line, in characters """ if length < self.last_line_len: - print ' ' * (self.last_line_len - length), - print '\r', + Print(' ' * (self.last_line_len - length), newline=False) + Print('\r', newline=False) self.last_line_len = length sys.stdout.flush() @@ -697,7 +401,7 @@ class Builder: Args: commit: Commit object that is being built brd: Board object that is being built - stage: Stage that we are at (distclean, config, build) + stage: Stage that we are at (mrproper, config, build) cwd: Directory where make should be run args: Arguments to pass to make kwargs: Arguments to pass to command.RunPipe() @@ -705,23 +409,22 @@ class Builder: cmd = [self.gnu_make] + list(args) result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=cwd, raise_on_error=False, **kwargs) + if self.verbose_build: + result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout + result.combined = '%s\n' % (' '.join(cmd)) + result.combined return result def ProcessResult(self, result): """Process the result of a build, showing progress information Args: - result: A CommandResult object + result: A CommandResult object, which indicates the result for + a single build """ col = terminal.Color() if result: target = result.brd.target - if result.return_code < 0: - self.active = False - command.StopAll() - return - self.upto += 1 if result.return_code != 0: self.fail += 1 @@ -729,6 +432,13 @@ class Builder: self.warned += 1 if result.already_done: self.already_done += 1 + if self._verbose: + Print('\r', newline=False) + self.ClearLine(0) + boards_selected = {target : result.brd} + self.ResetResultSummary(boards_selected) + self.ProduceResultSummary(result.commit_upto, self.commits, + boards_selected) else: target = '(starting)' @@ -751,8 +461,8 @@ class Builder: self.commit_count) name += target - print line + name, - length = 13 + len(name) + Print(line + name, newline=False) + length = 16 + len(name) self.ClearLine(length) def _GetOutputDir(self, commit_upto): @@ -763,12 +473,17 @@ class Builder: Args: commit_upto: Commit number to use (0..self.count-1) """ - commit = self.commits[commit_upto] - subject = commit.subject.translate(trans_valid_chars) - commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, - self.commit_count, commit.hash, subject[:20])) - output_dir = os.path.join(self.base_dir, commit_dir) - return output_dir + commit_dir = None + if self.commits: + commit = self.commits[commit_upto] + subject = commit.subject.translate(trans_valid_chars) + commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, + self.commit_count, commit.hash, subject[:20])) + elif not self.no_subdirs: + commit_dir = 'current' + if not commit_dir: + return self.base_dir + return os.path.join(self.base_dir, commit_dir) def GetBuildDir(self, commit_upto, target): """Get the name of the build directory for a commit number @@ -864,7 +579,7 @@ class Builder: try: size, type, name = line[:-1].split() except: - print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) + Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) continue if type in 'tTdDbB': # function names begin with '.' on 64-bit powerpc @@ -873,13 +588,78 @@ class Builder: sym[name] = sym.get(name, 0) + int(size, 16) return sym - def GetBuildOutcome(self, commit_upto, target, read_func_sizes): + def _ProcessConfig(self, fname): + """Read in a .config, autoconf.mk or autoconf.h file + + This function handles all config file types. It ignores comments and + any #defines which don't start with CONFIG_. + + Args: + fname: Filename to read + + Returns: + Dictionary: + key: Config name (e.g. CONFIG_DM) + value: Config value (e.g. 1) + """ + config = {} + if os.path.exists(fname): + with open(fname) as fd: + for line in fd: + line = line.strip() + if line.startswith('#define'): + values = line[8:].split(' ', 1) + if len(values) > 1: + key, value = values + else: + key = values[0] + value = '1' if self.squash_config_y else '' + if not key.startswith('CONFIG_'): + continue + elif not line or line[0] in ['#', '*', '/']: + continue + else: + key, value = line.split('=', 1) + if self.squash_config_y and value == 'y': + value = '1' + config[key] = value + return config + + def _ProcessEnvironment(self, fname): + """Read in a uboot.env file + + This function reads in environment variables from a file. + + Args: + fname: Filename to read + + Returns: + Dictionary: + key: environment variable (e.g. bootlimit) + value: value of environment variable (e.g. 1) + """ + environment = {} + if os.path.exists(fname): + with open(fname) as fd: + for line in fd.read().split('\0'): + try: + key, value = line.split('=', 1) + environment[key] = value + except ValueError: + # ignore lines we can't parse + pass + return environment + + def GetBuildOutcome(self, commit_upto, target, read_func_sizes, + read_config, read_environment): """Work out the outcome of a build. Args: commit_upto: Commit number to check (0..n-1) target: Target board to check read_func_sizes: True to read function size information + read_config: True to read .config and autoconf.h files + read_environment: True to read uboot.env files Returns: Outcome object @@ -888,6 +668,8 @@ class Builder: sizes_file = self.GetSizesFile(commit_upto, target) sizes = {} func_sizes = {} + config = {} + environment = {} if os.path.exists(done_file): with open(done_file, 'r') as fd: return_code = int(fd.readline()) @@ -931,35 +713,112 @@ class Builder: '') func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) - return Builder.Outcome(rc, err_lines, sizes, func_sizes) + if read_config: + output_dir = self.GetBuildDir(commit_upto, target) + for name in self.config_filenames: + fname = os.path.join(output_dir, name) + config[name] = self._ProcessConfig(fname) - return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) + if read_environment: + output_dir = self.GetBuildDir(commit_upto, target) + fname = os.path.join(output_dir, 'uboot.env') + environment = self._ProcessEnvironment(fname) - def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): + return Builder.Outcome(rc, err_lines, sizes, func_sizes, config, + environment) + + return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {}) + + def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, + read_config, read_environment): """Calculate a summary of the results of building a commit. Args: board_selected: Dict containing boards to summarise commit_upto: Commit number to summarize (0..self.count-1) read_func_sizes: True to read function size information + read_config: True to read .config and autoconf.h files + read_environment: True to read uboot.env files Returns: Tuple: Dict containing boards which passed building this commit. keyed by board.target - List containing a summary of error/warning lines + List containing a summary of error lines + Dict keyed by error line, containing a list of the Board + objects with that error + List containing a summary of warning lines + Dict keyed by error line, containing a list of the Board + objects with that warning + Dictionary keyed by board.target. Each value is a dictionary: + key: filename - e.g. '.config' + value is itself a dictionary: + key: config name + value: config value + Dictionary keyed by board.target. Each value is a dictionary: + key: environment variable + value: value of environment variable """ + def AddLine(lines_summary, lines_boards, line, board): + line = line.rstrip() + if line in lines_boards: + lines_boards[line].append(board) + else: + lines_boards[line] = [board] + lines_summary.append(line) + board_dict = {} err_lines_summary = [] + err_lines_boards = {} + warn_lines_summary = [] + warn_lines_boards = {} + config = {} + environment = {} for board in boards_selected.itervalues(): outcome = self.GetBuildOutcome(commit_upto, board.target, - read_func_sizes) + read_func_sizes, read_config, + read_environment) board_dict[board.target] = outcome - for err in outcome.err_lines: - if err and not err.rstrip() in err_lines_summary: - err_lines_summary.append(err.rstrip()) - return board_dict, err_lines_summary + last_func = None + last_was_warning = False + for line in outcome.err_lines: + if line: + if (self._re_function.match(line) or + self._re_files.match(line)): + last_func = line + else: + is_warning = self._re_warning.match(line) + is_note = self._re_note.match(line) + if is_warning or (last_was_warning and is_note): + if last_func: + AddLine(warn_lines_summary, warn_lines_boards, + last_func, board) + AddLine(warn_lines_summary, warn_lines_boards, + line, board) + else: + if last_func: + AddLine(err_lines_summary, err_lines_boards, + last_func, board) + AddLine(err_lines_summary, err_lines_boards, + line, board) + last_was_warning = is_warning + last_func = None + tconfig = Config(self.config_filenames, board.target) + for fname in self.config_filenames: + if outcome.config: + for key, value in outcome.config[fname].iteritems(): + tconfig.Add(fname, key, value) + config[board.target] = tconfig + + tenvironment = Environment(board.target) + if outcome.environment: + for key, value in outcome.environment.iteritems(): + tenvironment.Add(key, value) + environment[board.target] = tenvironment + + return (board_dict, err_lines_summary, err_lines_boards, + warn_lines_summary, warn_lines_boards, config, environment) def AddOutcome(self, board_dict, arch_list, changes, char, color): """Add an output to our list of outcomes for each architecture @@ -983,7 +842,7 @@ class Builder: arch = 'unknown' str = self.col.Color(color, ' ' + target) if not arch in done_arch: - str = self.col.Color(color, char) + ' ' + str + str = ' %s %s' % (self.col.Color(color, char), str) done_arch[arch] = True if not arch in arch_list: arch_list[arch] = str @@ -1012,8 +871,14 @@ class Builder: """ self._base_board_dict = {} for board in board_selected: - self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) + self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}, + {}) self._base_err_lines = [] + self._base_warn_lines = [] + self._base_err_line_boards = {} + self._base_warn_line_boards = {} + self._base_config = None + self._base_environment = None def PrintFuncSizeDetail(self, fname, old, new): grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 @@ -1047,20 +912,20 @@ class Builder: delta.reverse() args = [add, -remove, grow, -shrink, up, -down, up - down] - if max(args) == 0: + if max(args) == 0 and min(args) == 0: return args = [self.ColourNum(x) for x in args] indent = ' ' * 15 - print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % - tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) - print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', - 'delta') + Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % + tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) + Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', + 'delta')) for diff, name in delta: if diff: color = self.col.RED if diff > 0 else self.col.GREEN msg = '%s %-38s %7s %7s %+7d' % (indent, name, old.get(name, '-'), new.get(name,'-'), diff) - print self.col.Color(color, msg) + Print(msg, colour=color) def PrintSizeDetail(self, target_list, show_bloat): @@ -1085,11 +950,12 @@ class Builder: color = self.col.RED if diff > 0 else self.col.GREEN msg = ' %s %+d' % (name, diff) if not printed_target: - print '%10s %-15s:' % ('', result['_target']), + Print('%10s %-15s:' % ('', result['_target']), + newline=False) printed_target = True - print self.col.Color(color, msg), + Print(msg, colour=color, newline=False) if printed_target: - print + Print() if show_bloat: target = result['_target'] outcome = result['_outcome'] @@ -1194,19 +1060,21 @@ class Builder: color = self.col.RED if avg_diff > 0 else self.col.GREEN msg = ' %s %+1.1f' % (name, avg_diff) if not printed_arch: - print '%10s: (for %d/%d boards)' % (arch, count, - arch_count[arch]), + Print('%10s: (for %d/%d boards)' % (arch, count, + arch_count[arch]), newline=False) printed_arch = True - print self.col.Color(color, msg), + Print(msg, colour=color, newline=False) if printed_arch: - print + Print() if show_detail: self.PrintSizeDetail(target_list, show_bloat) def PrintResultSummary(self, board_selected, board_dict, err_lines, - show_sizes, show_detail, show_bloat): + err_line_boards, warn_lines, warn_line_boards, + config, environment, show_sizes, show_detail, + show_bloat, show_config, show_environment): """Compare results with the base results and display delta. Only boards mentioned in board_selected will be considered. This @@ -1221,10 +1089,111 @@ class Builder: commit, keyed by board.target. The value is an Outcome object. err_lines: A list of errors for this commit, or [] if there is none, or we don't want to print errors + err_line_boards: Dict keyed by error line, containing a list of + the Board objects with that error + warn_lines: A list of warnings for this commit, or [] if there is + none, or we don't want to print errors + warn_line_boards: Dict keyed by warning line, containing a list of + the Board objects with that warning + config: Dictionary keyed by filename - e.g. '.config'. Each + value is itself a dictionary: + key: config name + value: config value + environment: Dictionary keyed by environment variable, Each + value is the value of environment variable. show_sizes: Show image size deltas show_detail: Show detail for each board show_bloat: Show detail for each function + show_config: Show config changes + show_environment: Show environment changes """ + def _BoardList(line, line_boards): + """Helper function to get a line of boards containing a line + + Args: + line: Error line to search for + Return: + String containing a list of boards with that error line, or + '' if the user has not requested such a list + """ + if self._list_error_boards: + names = [] + for board in line_boards[line]: + if not board.target in names: + names.append(board.target) + names_str = '(%s) ' % ','.join(names) + else: + names_str = '' + return names_str + + def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, + char): + better_lines = [] + worse_lines = [] + for line in lines: + if line not in base_lines: + worse_lines.append(char + '+' + + _BoardList(line, line_boards) + line) + for line in base_lines: + if line not in lines: + better_lines.append(char + '-' + + _BoardList(line, base_line_boards) + line) + return better_lines, worse_lines + + def _CalcConfig(delta, name, config): + """Calculate configuration changes + + Args: + delta: Type of the delta, e.g. '+' + name: name of the file which changed (e.g. .config) + config: configuration change dictionary + key: config name + value: config value + Returns: + String containing the configuration changes which can be + printed + """ + out = '' + for key in sorted(config.keys()): + out += '%s=%s ' % (key, config[key]) + return '%s %s: %s' % (delta, name, out) + + def _AddConfig(lines, name, config_plus, config_minus, config_change): + """Add changes in configuration to a list + + Args: + lines: list to add to + name: config file name + config_plus: configurations added, dictionary + key: config name + value: config value + config_minus: configurations removed, dictionary + key: config name + value: config value + config_change: configurations changed, dictionary + key: config name + value: config value + """ + if config_plus: + lines.append(_CalcConfig('+', name, config_plus)) + if config_minus: + lines.append(_CalcConfig('-', name, config_minus)) + if config_change: + lines.append(_CalcConfig('c', name, config_change)) + + def _OutputConfigInfo(lines): + for line in lines: + if not line: + continue + if line[0] == '+': + col = self.col.GREEN + elif line[0] == '-': + col = self.col.RED + elif line[0] == 'c': + col = self.col.YELLOW + Print(' ' + line, newline=True, colour=col) + + better = [] # List of boards fixed since last commit worse = [] # List of new broken boards since last commit new = [] # List of boards that didn't exist last time @@ -1248,17 +1217,14 @@ class Builder: new.append(target) # Get a list of errors that have appeared, and disappeared - better_err = [] - worse_err = [] - for line in err_lines: - if line not in self._base_err_lines: - worse_err.append('+' + line) - for line in self._base_err_lines: - if line not in err_lines: - better_err.append('-' + line) + better_err, worse_err = _CalcErrorDelta(self._base_err_lines, + self._base_err_line_boards, err_lines, err_line_boards, '') + better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, + self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') # Display results by arch - if better or worse or unknown or new or worse_err or better_err: + if (better or worse or unknown or new or worse_err or better_err + or worse_warn or better_warn): arch_list = {} self.AddOutcome(board_selected, arch_list, better, '', self.col.GREEN) @@ -1269,19 +1235,162 @@ class Builder: self.AddOutcome(board_selected, arch_list, unknown, '?', self.col.MAGENTA) for arch, target_list in arch_list.iteritems(): - print '%10s: %s' % (arch, target_list) + Print('%10s: %s' % (arch, target_list)) + self._error_lines += 1 if better_err: - print self.col.Color(self.col.GREEN, '\n'.join(better_err)) + Print('\n'.join(better_err), colour=self.col.GREEN) + self._error_lines += 1 if worse_err: - print self.col.Color(self.col.RED, '\n'.join(worse_err)) + Print('\n'.join(worse_err), colour=self.col.RED) + self._error_lines += 1 + if better_warn: + Print('\n'.join(better_warn), colour=self.col.CYAN) + self._error_lines += 1 + if worse_warn: + Print('\n'.join(worse_warn), colour=self.col.MAGENTA) + self._error_lines += 1 if show_sizes: self.PrintSizeSummary(board_selected, board_dict, show_detail, show_bloat) + if show_environment and self._base_environment: + lines = [] + + for target in board_dict: + if target not in board_selected: + continue + + tbase = self._base_environment[target] + tenvironment = environment[target] + environment_plus = {} + environment_minus = {} + environment_change = {} + base = tbase.environment + for key, value in tenvironment.environment.iteritems(): + if key not in base: + environment_plus[key] = value + for key, value in base.iteritems(): + if key not in tenvironment.environment: + environment_minus[key] = value + for key, value in base.iteritems(): + new_value = tenvironment.environment.get(key) + if new_value and value != new_value: + desc = '%s -> %s' % (value, new_value) + environment_change[key] = desc + + _AddConfig(lines, target, environment_plus, environment_minus, + environment_change) + + _OutputConfigInfo(lines) + + if show_config and self._base_config: + summary = {} + arch_config_plus = {} + arch_config_minus = {} + arch_config_change = {} + arch_list = [] + + for target in board_dict: + if target not in board_selected: + continue + arch = board_selected[target].arch + if arch not in arch_list: + arch_list.append(arch) + + for arch in arch_list: + arch_config_plus[arch] = {} + arch_config_minus[arch] = {} + arch_config_change[arch] = {} + for name in self.config_filenames: + arch_config_plus[arch][name] = {} + arch_config_minus[arch][name] = {} + arch_config_change[arch][name] = {} + + for target in board_dict: + if target not in board_selected: + continue + + arch = board_selected[target].arch + + all_config_plus = {} + all_config_minus = {} + all_config_change = {} + tbase = self._base_config[target] + tconfig = config[target] + lines = [] + for name in self.config_filenames: + if not tconfig.config[name]: + continue + config_plus = {} + config_minus = {} + config_change = {} + base = tbase.config[name] + for key, value in tconfig.config[name].iteritems(): + if key not in base: + config_plus[key] = value + all_config_plus[key] = value + for key, value in base.iteritems(): + if key not in tconfig.config[name]: + config_minus[key] = value + all_config_minus[key] = value + for key, value in base.iteritems(): + new_value = tconfig.config.get(key) + if new_value and value != new_value: + desc = '%s -> %s' % (value, new_value) + config_change[key] = desc + all_config_change[key] = desc + + arch_config_plus[arch][name].update(config_plus) + arch_config_minus[arch][name].update(config_minus) + arch_config_change[arch][name].update(config_change) + + _AddConfig(lines, name, config_plus, config_minus, + config_change) + _AddConfig(lines, 'all', all_config_plus, all_config_minus, + all_config_change) + summary[target] = '\n'.join(lines) + + lines_by_target = {} + for target, lines in summary.iteritems(): + if lines in lines_by_target: + lines_by_target[lines].append(target) + else: + lines_by_target[lines] = [target] + + for arch in arch_list: + lines = [] + all_plus = {} + all_minus = {} + all_change = {} + for name in self.config_filenames: + all_plus.update(arch_config_plus[arch][name]) + all_minus.update(arch_config_minus[arch][name]) + all_change.update(arch_config_change[arch][name]) + _AddConfig(lines, name, arch_config_plus[arch][name], + arch_config_minus[arch][name], + arch_config_change[arch][name]) + _AddConfig(lines, 'all', all_plus, all_minus, all_change) + #arch_summary[target] = '\n'.join(lines) + if lines: + Print('%s:' % arch) + _OutputConfigInfo(lines) + + for lines, targets in lines_by_target.iteritems(): + if not lines: + continue + Print('%s :' % ' '.join(sorted(targets))) + _OutputConfigInfo(lines.split('\n')) + + # Save our updated information for the next call to this function self._base_board_dict = board_dict self._base_err_lines = err_lines + self._base_warn_lines = warn_lines + self._base_err_line_boards = err_line_boards + self._base_warn_line_boards = warn_line_boards + self._base_config = config + self._base_environment = environment # Get a list of boards that did not get built, if needed not_built = [] @@ -1289,12 +1398,27 @@ class Builder: if not board in board_dict: not_built.append(board) if not_built: - print "Boards not built (%d): %s" % (len(not_built), - ', '.join(not_built)) - + Print("Boards not built (%d): %s" % (len(not_built), + ', '.join(not_built))) + + def ProduceResultSummary(self, commit_upto, commits, board_selected): + (board_dict, err_lines, err_line_boards, warn_lines, + warn_line_boards, config, environment) = self.GetResultSummary( + board_selected, commit_upto, + read_func_sizes=self._show_bloat, + read_config=self._show_config, + read_environment=self._show_environment) + if commits: + msg = '%02d: %s' % (commit_upto + 1, + commits[commit_upto].subject) + Print(msg, colour=self.col.BLUE) + self.PrintResultSummary(board_selected, board_dict, + err_lines if self._show_errors else [], err_line_boards, + warn_lines if self._show_errors else [], warn_line_boards, + config, environment, self._show_sizes, self._show_detail, + self._show_bloat, self._show_config, self._show_environment) - def ShowSummary(self, commits, board_selected, show_errors, show_sizes, - show_detail, show_bloat): + def ShowSummary(self, commits, board_selected): """Show a build summary for U-Boot for a given board list. Reset the result summary, then repeatedly call GetResultSummary on @@ -1303,23 +1427,16 @@ class Builder: Args: commit: Commit objects to summarise board_selected: Dict containing boards to summarise - show_errors: Show errors that occured - show_sizes: Show size deltas - show_detail: Show detail for each board - show_bloat: Show detail for each function """ - self.commit_count = len(commits) + self.commit_count = len(commits) if commits else 1 self.commits = commits self.ResetResultSummary(board_selected) + self._error_lines = 0 for commit_upto in range(0, self.commit_count, self._step): - board_dict, err_lines = self.GetResultSummary(board_selected, - commit_upto, read_func_sizes=show_bloat) - msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) - print self.col.Color(self.col.BLUE, msg) - self.PrintResultSummary(board_selected, board_dict, - err_lines if show_errors else [], show_sizes, show_detail, - show_bloat) + self.ProduceResultSummary(commit_upto, commits, board_selected) + if not self._error_lines: + Print('(no errors to report)', colour=self.col.GREEN) def SetupBuild(self, board_selected, commits): @@ -1330,45 +1447,11 @@ class Builder: commits: Selected commits to build """ # First work out how many commits we will build - count = (len(commits) + self._step - 1) / self._step + count = (self.commit_count + self._step - 1) / self._step self.count = len(board_selected) * count self.upto = self.warned = self.fail = 0 self._timestamps = collections.deque() - def BuildBoardsForCommit(self, board_selected, keep_outputs): - """Build all boards for a single commit""" - self.SetupBuild(board_selected) - self.count = len(board_selected) - for brd in board_selected.itervalues(): - job = BuilderJob() - job.board = brd - job.commits = None - job.keep_outputs = keep_outputs - self.queue.put(brd) - - self.queue.join() - self.out_queue.join() - print - self.ClearLine(0) - - def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): - """Build all boards for all commits (non-incremental)""" - self.commit_count = len(commits) - - self.ResetResultSummary(board_selected) - for self.commit_upto in range(self.commit_count): - self.SelectCommit(commits[self.commit_upto]) - self.SelectOutputDir() - Mkdir(self.output_dir) - - self.BuildBoardsForCommit(board_selected, keep_outputs) - board_dict, err_lines = self.GetResultSummary() - self.PrintResultSummary(board_selected, board_dict, - err_lines if show_errors else []) - - if self.already_done: - print '%d builds already done' % self.already_done - def GetThreadDir(self, thread_num): """Get the directory path to the working dir for a thread. @@ -1377,40 +1460,44 @@ class Builder: """ return os.path.join(self._working_dir, '%02d' % thread_num) - def _PrepareThread(self, thread_num): + def _PrepareThread(self, thread_num, setup_git): """Prepare the working directory for a thread. This clones or fetches the repo into the thread's work directory. Args: thread_num: Thread number (0, 1, ...) + setup_git: True to set up a git repo clone """ thread_dir = self.GetThreadDir(thread_num) - Mkdir(thread_dir) + builderthread.Mkdir(thread_dir) git_dir = os.path.join(thread_dir, '.git') # Clone the repo if it doesn't already exist # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so # we have a private index but uses the origin repo's contents? - if self.git_dir: + if setup_git and self.git_dir: src_dir = os.path.abspath(self.git_dir) if os.path.exists(git_dir): gitutil.Fetch(git_dir, thread_dir) else: - print 'Cloning repo for thread %d' % thread_num + Print('\rCloning repo for thread %d' % thread_num, + newline=False) gitutil.Clone(src_dir, thread_dir) + Print('\r%s\r' % (' ' * 30), newline=False) - def _PrepareWorkingSpace(self, max_threads): + def _PrepareWorkingSpace(self, max_threads, setup_git): """Prepare the working directory for use. Set up the git repo for each thread. Args: max_threads: Maximum number of threads we expect to need. + setup_git: True to set up a git repo clone """ - Mkdir(self._working_dir) + builderthread.Mkdir(self._working_dir) for thread in range(max_threads): - self._PrepareThread(thread) + self._PrepareThread(thread, setup_git) def _PrepareOutputSpace(self): """Get the output directories ready to receive files. @@ -1419,47 +1506,66 @@ class Builder: create. Having left over directories is confusing when the user wants to check the output manually. """ + if not self.commits: + return dir_list = [] for commit_upto in range(self.commit_count): dir_list.append(self._GetOutputDir(commit_upto)) + to_remove = [] for dirname in glob.glob(os.path.join(self.base_dir, '*')): if dirname not in dir_list: + to_remove.append(dirname) + if to_remove: + Print('Removing %d old build directories' % len(to_remove), + newline=False) + for dirname in to_remove: shutil.rmtree(dirname) - def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): + def BuildBoards(self, commits, board_selected, keep_outputs, verbose): """Build all commits for a list of boards Args: commits: List of commits to be build, each a Commit object boards_selected: Dict of selected boards, key is target name, value is Board object - show_errors: True to show summarised error/warning info keep_outputs: True to save build output files + verbose: Display build results as they are completed + Returns: + Tuple containing: + - number of boards that failed to build + - number of boards that issued warnings """ - self.commit_count = len(commits) + self.commit_count = len(commits) if commits else 1 self.commits = commits + self._verbose = verbose self.ResetResultSummary(board_selected) - Mkdir(self.base_dir) - self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) + builderthread.Mkdir(self.base_dir, parents = True) + self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), + commits is not None) self._PrepareOutputSpace() + Print('\rStarting build...', newline=False) self.SetupBuild(board_selected, commits) self.ProcessResult(None) # Create jobs to build all commits for each board for brd in board_selected.itervalues(): - job = BuilderJob() + job = builderthread.BuilderJob() job.board = brd job.commits = commits job.keep_outputs = keep_outputs job.step = self._step self.queue.put(job) - # Wait until all jobs are started - self.queue.join() + term = threading.Thread(target=self.queue.join) + term.setDaemon(True) + term.start() + while term.isAlive(): + term.join(100) # Wait until we have processed all output self.out_queue.join() - print + Print() self.ClearLine(0) + return (self.fail, self.warned)