tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 import builderthread
  21 import command
  22 import gitutil
  23 import terminal
  24 from terminal import Print
  25 import toolchain
  26
  27 """
  28 Theory of Operation
  29
  30 Please see README for user documentation, and you should be familiar with
  31 that before trying to make sense of this.
  32
  33 Buildman works by keeping the machine as busy as possible, building different
  34 commits for different boards on multiple CPUs at once.
  35
  36 The source repo (self.git_dir) contains all the commits to be built. Each
  37 thread works on a single board at a time. It checks out the first commit,
  38 configures it for that board, then builds it. Then it checks out the next
  39 commit and builds it (typically without re-configuring). When it runs out
  40 of commits, it gets another job from the builder and starts again with that
  41 board.
  42
  43 Clearly the builder threads could work either way - they could check out a
  44 commit and then built it for all boards. Using separate directories for each
  45 commit/board pair they could leave their build product around afterwards
  46 also.
  47
  48 The intent behind building a single board for multiple commits, is to make
  49 use of incremental builds. Since each commit is built incrementally from
  50 the previous one, builds are faster. Reconfiguring for a different board
  51 removes all intermediate object files.
  52
  53 Many threads can be working at once, but each has its own working directory.
  54 When a thread finishes a build, it puts the output files into a result
  55 directory.
  56
  57 The base directory used by buildman is normally '../<branch>', i.e.
  58 a directory higher than the source repository and named after the branch
  59 being built.
  60
  61 Within the base directory, we have one subdirectory for each commit. Within
  62 that is one subdirectory for each board. Within that is the build output for
  63 that commit/board combination.
  64
  65 Buildman also create working directories for each thread, in a .bm-work/
  66 subdirectory in the base dir.
  67
  68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  69 'seaboard', and say that us-net has two commits. We will have directories
  70 like this:
  71
  72 us-net/             base directory
  73     01_of_02_g4ed4ebc_net--Add-tftp-speed-/
  74         sandbox/
  75             u-boot.bin
  76         seaboard/
  77             u-boot.bin
  78     02_of_02_g4ed4ebc_net--Check-tftp-comp/
  79         sandbox/
  80             u-boot.bin
  81         seaboard/
  82             u-boot.bin
  83     .bm-work/
  84         00/         working directory for thread 0 (contains source checkout)
  85             build/  build output
  86         01/         working directory for thread 1
  87             build/  build output
  88         ...
  89 u-boot/             source directory
  90     .git/           repository
  91 """
  92
  93 """Holds information about a particular error line we are outputing
  94
  95    char: Character representation: '+': error, '-': fixed error, 'w+': warning,
  96        'w-' = fixed warning
  97    boards: List of Board objects which have line in the error/warning output
  98    errline: The text of the error line
  99 """
 100 ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
 101
 102 # Possible build outcomes
 103 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 104
 105 # Translate a commit subject into a valid filename (and handle unicode)
 106 trans_valid_chars = str.maketrans('/: ', '---')
 107
 108 BASE_CONFIG_FILENAMES = [
 109     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 110 ]
 111
 112 EXTRA_CONFIG_FILENAMES = [
 113     '.config', '.config-spl', '.config-tpl',
 114     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 115     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 116 ]
 117
 118 class Config:
 119     """Holds information about configuration settings for a board."""
 120     def __init__(self, config_filename, target):
 121         self.target = target
 122         self.config = {}
 123         for fname in config_filename:
 124             self.config[fname] = {}
 125
 126     def Add(self, fname, key, value):
 127         self.config[fname][key] = value
 128
 129     def __hash__(self):
 130         val = 0
 131         for fname in self.config:
 132             for key, value in self.config[fname].items():
 133                 print(key, value)
 134                 val = val ^ hash(key) & hash(value)
 135         return val
 136
 137 class Environment:
 138     """Holds information about environment variables for a board."""
 139     def __init__(self, target):
 140         self.target = target
 141         self.environment = {}
 142
 143     def Add(self, key, value):
 144         self.environment[key] = value
 145
 146 class Builder:
 147     """Class for building U-Boot for a particular commit.
 148
 149     Public members: (many should ->private)
 150         already_done: Number of builds already completed
 151         base_dir: Base directory to use for builder
 152         checkout: True to check out source, False to skip that step.
 153             This is used for testing.
 154         col: terminal.Color() object
 155         count: Number of commits to build
 156         do_make: Method to call to invoke Make
 157         fail: Number of builds that failed due to error
 158         force_build: Force building even if a build already exists
 159         force_config_on_failure: If a commit fails for a board, disable
 160             incremental building for the next commit we build for that
 161             board, so that we will see all warnings/errors again.
 162         force_build_failures: If a previously-built build (i.e. built on
 163             a previous run of buildman) is marked as failed, rebuild it.
 164         git_dir: Git directory containing source repository
 165         num_jobs: Number of jobs to run at once (passed to make as -j)
 166         num_threads: Number of builder threads to run
 167         out_queue: Queue of results to process
 168         re_make_err: Compiled regular expression for ignore_lines
 169         queue: Queue of jobs to run
 170         threads: List of active threads
 171         toolchains: Toolchains object to use for building
 172         upto: Current commit number we are building (0.count-1)
 173         warned: Number of builds that produced at least one warning
 174         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 175             incremental building, where buildman reconfigures on the first
 176             commit for a baord, and then just does an incremental build for
 177             the following commits. In fact buildman will reconfigure and
 178             retry for any failing commits, so generally the only effect of
 179             this option is to slow things down.
 180         in_tree: Build U-Boot in-tree instead of specifying an output
 181             directory separate from the source code. This option is really
 182             only useful for testing in-tree builds.
 183         work_in_output: Use the output directory as the work directory and
 184             don't write to a separate output directory.
 185
 186     Private members:
 187         _base_board_dict: Last-summarised Dict of boards
 188         _base_err_lines: Last-summarised list of errors
 189         _base_warn_lines: Last-summarised list of warnings
 190         _build_period_us: Time taken for a single build (float object).
 191         _complete_delay: Expected delay until completion (timedelta)
 192         _next_delay_update: Next time we plan to display a progress update
 193                 (datatime)
 194         _show_unknown: Show unknown boards (those not built) in summary
 195         _timestamps: List of timestamps for the completion of the last
 196             last _timestamp_count builds. Each is a datetime object.
 197         _timestamp_count: Number of timestamps to keep in our list.
 198         _working_dir: Base working directory containing all threads
 199     """
 200     class Outcome:
 201         """Records a build outcome for a single make invocation
 202
 203         Public Members:
 204             rc: Outcome value (OUTCOME_...)
 205             err_lines: List of error lines or [] if none
 206             sizes: Dictionary of image size information, keyed by filename
 207                 - Each value is itself a dictionary containing
 208                     values for 'text', 'data' and 'bss', being the integer
 209                     size in bytes of each section.
 210             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 211                     value is itself a dictionary:
 212                         key: function name
 213                         value: Size of function in bytes
 214             config: Dictionary keyed by filename - e.g. '.config'. Each
 215                     value is itself a dictionary:
 216                         key: config name
 217                         value: config value
 218             environment: Dictionary keyed by environment variable, Each
 219                      value is the value of environment variable.
 220         """
 221         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 222                      environment):
 223             self.rc = rc
 224             self.err_lines = err_lines
 225             self.sizes = sizes
 226             self.func_sizes = func_sizes
 227             self.config = config
 228             self.environment = environment
 229
 230     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 231                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 232                  no_subdirs=False, full_path=False, verbose_build=False,
 233                  incremental=False, per_board_out_dir=False,
 234                  config_only=False, squash_config_y=False,
 235                  warnings_as_errors=False, work_in_output=False):
 236         """Create a new Builder object
 237
 238         Args:
 239             toolchains: Toolchains object to use for building
 240             base_dir: Base directory to use for builder
 241             git_dir: Git directory containing source repository
 242             num_threads: Number of builder threads to run
 243             num_jobs: Number of jobs to run at once (passed to make as -j)
 244             gnu_make: the command name of GNU Make.
 245             checkout: True to check out source, False to skip that step.
 246                 This is used for testing.
 247             show_unknown: Show unknown boards (those not built) in summary
 248             step: 1 to process every commit, n to process every nth commit
 249             no_subdirs: Don't create subdirectories when building current
 250                 source for a single board
 251             full_path: Return the full path in CROSS_COMPILE and don't set
 252                 PATH
 253             verbose_build: Run build with V=1 and don't use 'make -s'
 254             incremental: Always perform incremental builds; don't run make
 255                 mrproper when configuring
 256             per_board_out_dir: Build in a separate persistent directory per
 257                 board rather than a thread-specific directory
 258             config_only: Only configure each build, don't build it
 259             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 260             warnings_as_errors: Treat all compiler warnings as errors
 261             work_in_output: Use the output directory as the work directory and
 262                 don't write to a separate output directory.
 263         """
 264         self.toolchains = toolchains
 265         self.base_dir = base_dir
 266         if work_in_output:
 267             self._working_dir = base_dir
 268         else:
 269             self._working_dir = os.path.join(base_dir, '.bm-work')
 270         self.threads = []
 271         self.do_make = self.Make
 272         self.gnu_make = gnu_make
 273         self.checkout = checkout
 274         self.num_threads = num_threads
 275         self.num_jobs = num_jobs
 276         self.already_done = 0
 277         self.force_build = False
 278         self.git_dir = git_dir
 279         self._show_unknown = show_unknown
 280         self._timestamp_count = 10
 281         self._build_period_us = None
 282         self._complete_delay = None
 283         self._next_delay_update = datetime.now()
 284         self.force_config_on_failure = True
 285         self.force_build_failures = False
 286         self.force_reconfig = False
 287         self._step = step
 288         self.in_tree = False
 289         self._error_lines = 0
 290         self.no_subdirs = no_subdirs
 291         self.full_path = full_path
 292         self.verbose_build = verbose_build
 293         self.config_only = config_only
 294         self.squash_config_y = squash_config_y
 295         self.config_filenames = BASE_CONFIG_FILENAMES
 296         self.work_in_output = work_in_output
 297         if not self.squash_config_y:
 298             self.config_filenames += EXTRA_CONFIG_FILENAMES
 299
 300         self.warnings_as_errors = warnings_as_errors
 301         self.col = terminal.Color()
 302
 303         self._re_function = re.compile('(.*): In function.*')
 304         self._re_files = re.compile('In file included from.*')
 305         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 306         self._re_dtb_warning = re.compile('(.*): Warning .*')
 307         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 308
 309         self.queue = queue.Queue()
 310         self.out_queue = queue.Queue()
 311         for i in range(self.num_threads):
 312             t = builderthread.BuilderThread(self, i, incremental,
 313                     per_board_out_dir)
 314             t.setDaemon(True)
 315             t.start()
 316             self.threads.append(t)
 317
 318         t = builderthread.ResultThread(self)
 319         t.setDaemon(True)
 320         t.start()
 321         self.threads.append(t)
 322
 323         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 324         self.re_make_err = re.compile('|'.join(ignore_lines))
 325
 326         # Handle existing graceful with SIGINT / Ctrl-C
 327         signal.signal(signal.SIGINT, self.signal_handler)
 328
 329     def __del__(self):
 330         """Get rid of all threads created by the builder"""
 331         for t in self.threads:
 332             del t
 333
 334     def signal_handler(self, signal, frame):
 335         sys.exit(1)
 336
 337     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 338                           show_detail=False, show_bloat=False,
 339                           list_error_boards=False, show_config=False,
 340                           show_environment=False):
 341         """Setup display options for the builder.
 342
 343         show_errors: True to show summarised error/warning info
 344         show_sizes: Show size deltas
 345         show_detail: Show size delta detail for each board if show_sizes
 346         show_bloat: Show detail for each function
 347         list_error_boards: Show the boards which caused each error/warning
 348         show_config: Show config deltas
 349         show_environment: Show environment deltas
 350         """
 351         self._show_errors = show_errors
 352         self._show_sizes = show_sizes
 353         self._show_detail = show_detail
 354         self._show_bloat = show_bloat
 355         self._list_error_boards = list_error_boards
 356         self._show_config = show_config
 357         self._show_environment = show_environment
 358
 359     def _AddTimestamp(self):
 360         """Add a new timestamp to the list and record the build period.
 361
 362         The build period is the length of time taken to perform a single
 363         build (one board, one commit).
 364         """
 365         now = datetime.now()
 366         self._timestamps.append(now)
 367         count = len(self._timestamps)
 368         delta = self._timestamps[-1] - self._timestamps[0]
 369         seconds = delta.total_seconds()
 370
 371         # If we have enough data, estimate build period (time taken for a
 372         # single build) and therefore completion time.
 373         if count > 1 and self._next_delay_update < now:
 374             self._next_delay_update = now + timedelta(seconds=2)
 375             if seconds > 0:
 376                 self._build_period = float(seconds) / count
 377                 todo = self.count - self.upto
 378                 self._complete_delay = timedelta(microseconds=
 379                         self._build_period * todo * 1000000)
 380                 # Round it
 381                 self._complete_delay -= timedelta(
 382                         microseconds=self._complete_delay.microseconds)
 383
 384         if seconds > 60:
 385             self._timestamps.popleft()
 386             count -= 1
 387
 388     def SelectCommit(self, commit, checkout=True):
 389         """Checkout the selected commit for this build
 390         """
 391         self.commit = commit
 392         if checkout and self.checkout:
 393             gitutil.Checkout(commit.hash)
 394
 395     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 396         """Run make
 397
 398         Args:
 399             commit: Commit object that is being built
 400             brd: Board object that is being built
 401             stage: Stage that we are at (mrproper, config, build)
 402             cwd: Directory where make should be run
 403             args: Arguments to pass to make
 404             kwargs: Arguments to pass to command.RunPipe()
 405         """
 406         cmd = [self.gnu_make] + list(args)
 407         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 408                 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
 409         if self.verbose_build:
 410             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 411             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 412         return result
 413
 414     def ProcessResult(self, result):
 415         """Process the result of a build, showing progress information
 416
 417         Args:
 418             result: A CommandResult object, which indicates the result for
 419                     a single build
 420         """
 421         col = terminal.Color()
 422         if result:
 423             target = result.brd.target
 424
 425             self.upto += 1
 426             if result.return_code != 0:
 427                 self.fail += 1
 428             elif result.stderr:
 429                 self.warned += 1
 430             if result.already_done:
 431                 self.already_done += 1
 432             if self._verbose:
 433                 terminal.PrintClear()
 434                 boards_selected = {target : result.brd}
 435                 self.ResetResultSummary(boards_selected)
 436                 self.ProduceResultSummary(result.commit_upto, self.commits,
 437                                           boards_selected)
 438         else:
 439             target = '(starting)'
 440
 441         # Display separate counts for ok, warned and fail
 442         ok = self.upto - self.warned - self.fail
 443         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 444         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 445         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 446
 447         line += ' /%-5d  ' % self.count
 448         remaining = self.count - self.upto
 449         if remaining:
 450             line += self.col.Color(self.col.MAGENTA, ' -%-5d  ' % remaining)
 451         else:
 452             line += ' ' * 8
 453
 454         # Add our current completion time estimate
 455         self._AddTimestamp()
 456         if self._complete_delay:
 457             line += '%s  : ' % self._complete_delay
 458
 459         line += target
 460         terminal.PrintClear()
 461         Print(line, newline=False, limit_to_line=True)
 462
 463     def _GetOutputDir(self, commit_upto):
 464         """Get the name of the output directory for a commit number
 465
 466         The output directory is typically .../<branch>/<commit>.
 467
 468         Args:
 469             commit_upto: Commit number to use (0..self.count-1)
 470         """
 471         commit_dir = None
 472         if self.commits:
 473             commit = self.commits[commit_upto]
 474             subject = commit.subject.translate(trans_valid_chars)
 475             # See _GetOutputSpaceRemovals() which parses this name
 476             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
 477                     self.commit_count, commit.hash, subject[:20]))
 478         elif not self.no_subdirs:
 479             commit_dir = 'current'
 480         if not commit_dir:
 481             return self.base_dir
 482         return os.path.join(self.base_dir, commit_dir)
 483
 484     def GetBuildDir(self, commit_upto, target):
 485         """Get the name of the build directory for a commit number
 486
 487         The build directory is typically .../<branch>/<commit>/<target>.
 488
 489         Args:
 490             commit_upto: Commit number to use (0..self.count-1)
 491             target: Target name
 492         """
 493         output_dir = self._GetOutputDir(commit_upto)
 494         return os.path.join(output_dir, target)
 495
 496     def GetDoneFile(self, commit_upto, target):
 497         """Get the name of the done file for a commit number
 498
 499         Args:
 500             commit_upto: Commit number to use (0..self.count-1)
 501             target: Target name
 502         """
 503         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 504
 505     def GetSizesFile(self, commit_upto, target):
 506         """Get the name of the sizes file for a commit number
 507
 508         Args:
 509             commit_upto: Commit number to use (0..self.count-1)
 510             target: Target name
 511         """
 512         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 513
 514     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 515         """Get the name of the funcsizes file for a commit number and ELF file
 516
 517         Args:
 518             commit_upto: Commit number to use (0..self.count-1)
 519             target: Target name
 520             elf_fname: Filename of elf image
 521         """
 522         return os.path.join(self.GetBuildDir(commit_upto, target),
 523                             '%s.sizes' % elf_fname.replace('/', '-'))
 524
 525     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 526         """Get the name of the objdump file for a commit number and ELF file
 527
 528         Args:
 529             commit_upto: Commit number to use (0..self.count-1)
 530             target: Target name
 531             elf_fname: Filename of elf image
 532         """
 533         return os.path.join(self.GetBuildDir(commit_upto, target),
 534                             '%s.objdump' % elf_fname.replace('/', '-'))
 535
 536     def GetErrFile(self, commit_upto, target):
 537         """Get the name of the err file for a commit number
 538
 539         Args:
 540             commit_upto: Commit number to use (0..self.count-1)
 541             target: Target name
 542         """
 543         output_dir = self.GetBuildDir(commit_upto, target)
 544         return os.path.join(output_dir, 'err')
 545
 546     def FilterErrors(self, lines):
 547         """Filter out errors in which we have no interest
 548
 549         We should probably use map().
 550
 551         Args:
 552             lines: List of error lines, each a string
 553         Returns:
 554             New list with only interesting lines included
 555         """
 556         out_lines = []
 557         for line in lines:
 558             if not self.re_make_err.search(line):
 559                 out_lines.append(line)
 560         return out_lines
 561
 562     def ReadFuncSizes(self, fname, fd):
 563         """Read function sizes from the output of 'nm'
 564
 565         Args:
 566             fd: File containing data to read
 567             fname: Filename we are reading from (just for errors)
 568
 569         Returns:
 570             Dictionary containing size of each function in bytes, indexed by
 571             function name.
 572         """
 573         sym = {}
 574         for line in fd.readlines():
 575             try:
 576                 if line.strip():
 577                     size, type, name = line[:-1].split()
 578             except:
 579                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 580                 continue
 581             if type in 'tTdDbB':
 582                 # function names begin with '.' on 64-bit powerpc
 583                 if '.' in name[1:]:
 584                     name = 'static.' + name.split('.')[0]
 585                 sym[name] = sym.get(name, 0) + int(size, 16)
 586         return sym
 587
 588     def _ProcessConfig(self, fname):
 589         """Read in a .config, autoconf.mk or autoconf.h file
 590
 591         This function handles all config file types. It ignores comments and
 592         any #defines which don't start with CONFIG_.
 593
 594         Args:
 595             fname: Filename to read
 596
 597         Returns:
 598             Dictionary:
 599                 key: Config name (e.g. CONFIG_DM)
 600                 value: Config value (e.g. 1)
 601         """
 602         config = {}
 603         if os.path.exists(fname):
 604             with open(fname) as fd:
 605                 for line in fd:
 606                     line = line.strip()
 607                     if line.startswith('#define'):
 608                         values = line[8:].split(' ', 1)
 609                         if len(values) > 1:
 610                             key, value = values
 611                         else:
 612                             key = values[0]
 613                             value = '1' if self.squash_config_y else ''
 614                         if not key.startswith('CONFIG_'):
 615                             continue
 616                     elif not line or line[0] in ['#', '*', '/']:
 617                         continue
 618                     else:
 619                         key, value = line.split('=', 1)
 620                     if self.squash_config_y and value == 'y':
 621                         value = '1'
 622                     config[key] = value
 623         return config
 624
 625     def _ProcessEnvironment(self, fname):
 626         """Read in a uboot.env file
 627
 628         This function reads in environment variables from a file.
 629
 630         Args:
 631             fname: Filename to read
 632
 633         Returns:
 634             Dictionary:
 635                 key: environment variable (e.g. bootlimit)
 636                 value: value of environment variable (e.g. 1)
 637         """
 638         environment = {}
 639         if os.path.exists(fname):
 640             with open(fname) as fd:
 641                 for line in fd.read().split('\0'):
 642                     try:
 643                         key, value = line.split('=', 1)
 644                         environment[key] = value
 645                     except ValueError:
 646                         # ignore lines we can't parse
 647                         pass
 648         return environment
 649
 650     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 651                         read_config, read_environment):
 652         """Work out the outcome of a build.
 653
 654         Args:
 655             commit_upto: Commit number to check (0..n-1)
 656             target: Target board to check
 657             read_func_sizes: True to read function size information
 658             read_config: True to read .config and autoconf.h files
 659             read_environment: True to read uboot.env files
 660
 661         Returns:
 662             Outcome object
 663         """
 664         done_file = self.GetDoneFile(commit_upto, target)
 665         sizes_file = self.GetSizesFile(commit_upto, target)
 666         sizes = {}
 667         func_sizes = {}
 668         config = {}
 669         environment = {}
 670         if os.path.exists(done_file):
 671             with open(done_file, 'r') as fd:
 672                 try:
 673                     return_code = int(fd.readline())
 674                 except ValueError:
 675                     # The file may be empty due to running out of disk space.
 676                     # Try a rebuild
 677                     return_code = 1
 678                 err_lines = []
 679                 err_file = self.GetErrFile(commit_upto, target)
 680                 if os.path.exists(err_file):
 681                     with open(err_file, 'r') as fd:
 682                         err_lines = self.FilterErrors(fd.readlines())
 683
 684                 # Decide whether the build was ok, failed or created warnings
 685                 if return_code:
 686                     rc = OUTCOME_ERROR
 687                 elif len(err_lines):
 688                     rc = OUTCOME_WARNING
 689                 else:
 690                     rc = OUTCOME_OK
 691
 692                 # Convert size information to our simple format
 693                 if os.path.exists(sizes_file):
 694                     with open(sizes_file, 'r') as fd:
 695                         for line in fd.readlines():
 696                             values = line.split()
 697                             rodata = 0
 698                             if len(values) > 6:
 699                                 rodata = int(values[6], 16)
 700                             size_dict = {
 701                                 'all' : int(values[0]) + int(values[1]) +
 702                                         int(values[2]),
 703                                 'text' : int(values[0]) - rodata,
 704                                 'data' : int(values[1]),
 705                                 'bss' : int(values[2]),
 706                                 'rodata' : rodata,
 707                             }
 708                             sizes[values[5]] = size_dict
 709
 710             if read_func_sizes:
 711                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 712                 for fname in glob.glob(pattern):
 713                     with open(fname, 'r') as fd:
 714                         dict_name = os.path.basename(fname).replace('.sizes',
 715                                                                     '')
 716                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 717
 718             if read_config:
 719                 output_dir = self.GetBuildDir(commit_upto, target)
 720                 for name in self.config_filenames:
 721                     fname = os.path.join(output_dir, name)
 722                     config[name] = self._ProcessConfig(fname)
 723
 724             if read_environment:
 725                 output_dir = self.GetBuildDir(commit_upto, target)
 726                 fname = os.path.join(output_dir, 'uboot.env')
 727                 environment = self._ProcessEnvironment(fname)
 728
 729             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 730                                    environment)
 731
 732         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 733
 734     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 735                          read_config, read_environment):
 736         """Calculate a summary of the results of building a commit.
 737
 738         Args:
 739             board_selected: Dict containing boards to summarise
 740             commit_upto: Commit number to summarize (0..self.count-1)
 741             read_func_sizes: True to read function size information
 742             read_config: True to read .config and autoconf.h files
 743             read_environment: True to read uboot.env files
 744
 745         Returns:
 746             Tuple:
 747                 Dict containing boards which passed building this commit.
 748                     keyed by board.target
 749                 List containing a summary of error lines
 750                 Dict keyed by error line, containing a list of the Board
 751                     objects with that error
 752                 List containing a summary of warning lines
 753                 Dict keyed by error line, containing a list of the Board
 754                     objects with that warning
 755                 Dictionary keyed by board.target. Each value is a dictionary:
 756                     key: filename - e.g. '.config'
 757                     value is itself a dictionary:
 758                         key: config name
 759                         value: config value
 760                 Dictionary keyed by board.target. Each value is a dictionary:
 761                     key: environment variable
 762                     value: value of environment variable
 763         """
 764         def AddLine(lines_summary, lines_boards, line, board):
 765             line = line.rstrip()
 766             if line in lines_boards:
 767                 lines_boards[line].append(board)
 768             else:
 769                 lines_boards[line] = [board]
 770                 lines_summary.append(line)
 771
 772         board_dict = {}
 773         err_lines_summary = []
 774         err_lines_boards = {}
 775         warn_lines_summary = []
 776         warn_lines_boards = {}
 777         config = {}
 778         environment = {}
 779
 780         for board in boards_selected.values():
 781             outcome = self.GetBuildOutcome(commit_upto, board.target,
 782                                            read_func_sizes, read_config,
 783                                            read_environment)
 784             board_dict[board.target] = outcome
 785             last_func = None
 786             last_was_warning = False
 787             for line in outcome.err_lines:
 788                 if line:
 789                     if (self._re_function.match(line) or
 790                             self._re_files.match(line)):
 791                         last_func = line
 792                     else:
 793                         is_warning = (self._re_warning.match(line) or
 794                                       self._re_dtb_warning.match(line))
 795                         is_note = self._re_note.match(line)
 796                         if is_warning or (last_was_warning and is_note):
 797                             if last_func:
 798                                 AddLine(warn_lines_summary, warn_lines_boards,
 799                                         last_func, board)
 800                             AddLine(warn_lines_summary, warn_lines_boards,
 801                                     line, board)
 802                         else:
 803                             if last_func:
 804                                 AddLine(err_lines_summary, err_lines_boards,
 805                                         last_func, board)
 806                             AddLine(err_lines_summary, err_lines_boards,
 807                                     line, board)
 808                         last_was_warning = is_warning
 809                         last_func = None
 810             tconfig = Config(self.config_filenames, board.target)
 811             for fname in self.config_filenames:
 812                 if outcome.config:
 813                     for key, value in outcome.config[fname].items():
 814                         tconfig.Add(fname, key, value)
 815             config[board.target] = tconfig
 816
 817             tenvironment = Environment(board.target)
 818             if outcome.environment:
 819                 for key, value in outcome.environment.items():
 820                     tenvironment.Add(key, value)
 821             environment[board.target] = tenvironment
 822
 823         return (board_dict, err_lines_summary, err_lines_boards,
 824                 warn_lines_summary, warn_lines_boards, config, environment)
 825
 826     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 827         """Add an output to our list of outcomes for each architecture
 828
 829         This simple function adds failing boards (changes) to the
 830         relevant architecture string, so we can print the results out
 831         sorted by architecture.
 832
 833         Args:
 834              board_dict: Dict containing all boards
 835              arch_list: Dict keyed by arch name. Value is a string containing
 836                     a list of board names which failed for that arch.
 837              changes: List of boards to add to arch_list
 838              color: terminal.Colour object
 839         """
 840         done_arch = {}
 841         for target in changes:
 842             if target in board_dict:
 843                 arch = board_dict[target].arch
 844             else:
 845                 arch = 'unknown'
 846             str = self.col.Color(color, ' ' + target)
 847             if not arch in done_arch:
 848                 str = ' %s  %s' % (self.col.Color(color, char), str)
 849                 done_arch[arch] = True
 850             if not arch in arch_list:
 851                 arch_list[arch] = str
 852             else:
 853                 arch_list[arch] += str
 854
 855
 856     def ColourNum(self, num):
 857         color = self.col.RED if num > 0 else self.col.GREEN
 858         if num == 0:
 859             return '0'
 860         return self.col.Color(color, str(num))
 861
 862     def ResetResultSummary(self, board_selected):
 863         """Reset the results summary ready for use.
 864
 865         Set up the base board list to be all those selected, and set the
 866         error lines to empty.
 867
 868         Following this, calls to PrintResultSummary() will use this
 869         information to work out what has changed.
 870
 871         Args:
 872             board_selected: Dict containing boards to summarise, keyed by
 873                 board.target
 874         """
 875         self._base_board_dict = {}
 876         for board in board_selected:
 877             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 878                                                            {})
 879         self._base_err_lines = []
 880         self._base_warn_lines = []
 881         self._base_err_line_boards = {}
 882         self._base_warn_line_boards = {}
 883         self._base_config = None
 884         self._base_environment = None
 885
 886     def PrintFuncSizeDetail(self, fname, old, new):
 887         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 888         delta, common = [], {}
 889
 890         for a in old:
 891             if a in new:
 892                 common[a] = 1
 893
 894         for name in old:
 895             if name not in common:
 896                 remove += 1
 897                 down += old[name]
 898                 delta.append([-old[name], name])
 899
 900         for name in new:
 901             if name not in common:
 902                 add += 1
 903                 up += new[name]
 904                 delta.append([new[name], name])
 905
 906         for name in common:
 907                 diff = new.get(name, 0) - old.get(name, 0)
 908                 if diff > 0:
 909                     grow, up = grow + 1, up + diff
 910                 elif diff < 0:
 911                     shrink, down = shrink + 1, down - diff
 912                 delta.append([diff, name])
 913
 914         delta.sort()
 915         delta.reverse()
 916
 917         args = [add, -remove, grow, -shrink, up, -down, up - down]
 918         if max(args) == 0 and min(args) == 0:
 919             return
 920         args = [self.ColourNum(x) for x in args]
 921         indent = ' ' * 15
 922         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
 923               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
 924         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
 925                                          'delta'))
 926         for diff, name in delta:
 927             if diff:
 928                 color = self.col.RED if diff > 0 else self.col.GREEN
 929                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
 930                         old.get(name, '-'), new.get(name,'-'), diff)
 931                 Print(msg, colour=color)
 932
 933
 934     def PrintSizeDetail(self, target_list, show_bloat):
 935         """Show details size information for each board
 936
 937         Args:
 938             target_list: List of targets, each a dict containing:
 939                     'target': Target name
 940                     'total_diff': Total difference in bytes across all areas
 941                     <part_name>: Difference for that part
 942             show_bloat: Show detail for each function
 943         """
 944         targets_by_diff = sorted(target_list, reverse=True,
 945         key=lambda x: x['_total_diff'])
 946         for result in targets_by_diff:
 947             printed_target = False
 948             for name in sorted(result):
 949                 diff = result[name]
 950                 if name.startswith('_'):
 951                     continue
 952                 if diff != 0:
 953                     color = self.col.RED if diff > 0 else self.col.GREEN
 954                 msg = ' %s %+d' % (name, diff)
 955                 if not printed_target:
 956                     Print('%10s  %-15s:' % ('', result['_target']),
 957                           newline=False)
 958                     printed_target = True
 959                 Print(msg, colour=color, newline=False)
 960             if printed_target:
 961                 Print()
 962                 if show_bloat:
 963                     target = result['_target']
 964                     outcome = result['_outcome']
 965                     base_outcome = self._base_board_dict[target]
 966                     for fname in outcome.func_sizes:
 967                         self.PrintFuncSizeDetail(fname,
 968                                                  base_outcome.func_sizes[fname],
 969                                                  outcome.func_sizes[fname])
 970
 971
 972     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
 973                          show_bloat):
 974         """Print a summary of image sizes broken down by section.
 975
 976         The summary takes the form of one line per architecture. The
 977         line contains deltas for each of the sections (+ means the section
 978         got bigger, - means smaller). The numbers are the average number
 979         of bytes that a board in this section increased by.
 980
 981         For example:
 982            powerpc: (622 boards)   text -0.0
 983           arm: (285 boards)   text -0.0
 984           nds32: (3 boards)   text -8.0
 985
 986         Args:
 987             board_selected: Dict containing boards to summarise, keyed by
 988                 board.target
 989             board_dict: Dict containing boards for which we built this
 990                 commit, keyed by board.target. The value is an Outcome object.
 991             show_detail: Show size delta detail for each board
 992             show_bloat: Show detail for each function
 993         """
 994         arch_list = {}
 995         arch_count = {}
 996
 997         # Calculate changes in size for different image parts
 998         # The previous sizes are in Board.sizes, for each board
 999         for target in board_dict:
1000             if target not in board_selected:
1001                 continue
1002             base_sizes = self._base_board_dict[target].sizes
1003             outcome = board_dict[target]
1004             sizes = outcome.sizes
1005
1006             # Loop through the list of images, creating a dict of size
1007             # changes for each image/part. We end up with something like
1008             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1009             # which means that U-Boot data increased by 5 bytes and SPL
1010             # text decreased by 4.
1011             err = {'_target' : target}
1012             for image in sizes:
1013                 if image in base_sizes:
1014                     base_image = base_sizes[image]
1015                     # Loop through the text, data, bss parts
1016                     for part in sorted(sizes[image]):
1017                         diff = sizes[image][part] - base_image[part]
1018                         col = None
1019                         if diff:
1020                             if image == 'u-boot':
1021                                 name = part
1022                             else:
1023                                 name = image + ':' + part
1024                             err[name] = diff
1025             arch = board_selected[target].arch
1026             if not arch in arch_count:
1027                 arch_count[arch] = 1
1028             else:
1029                 arch_count[arch] += 1
1030             if not sizes:
1031                 pass    # Only add to our list when we have some stats
1032             elif not arch in arch_list:
1033                 arch_list[arch] = [err]
1034             else:
1035                 arch_list[arch].append(err)
1036
1037         # We now have a list of image size changes sorted by arch
1038         # Print out a summary of these
1039         for arch, target_list in arch_list.items():
1040             # Get total difference for each type
1041             totals = {}
1042             for result in target_list:
1043                 total = 0
1044                 for name, diff in result.items():
1045                     if name.startswith('_'):
1046                         continue
1047                     total += diff
1048                     if name in totals:
1049                         totals[name] += diff
1050                     else:
1051                         totals[name] = diff
1052                 result['_total_diff'] = total
1053                 result['_outcome'] = board_dict[result['_target']]
1054
1055             count = len(target_list)
1056             printed_arch = False
1057             for name in sorted(totals):
1058                 diff = totals[name]
1059                 if diff:
1060                     # Display the average difference in this name for this
1061                     # architecture
1062                     avg_diff = float(diff) / count
1063                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1064                     msg = ' %s %+1.1f' % (name, avg_diff)
1065                     if not printed_arch:
1066                         Print('%10s: (for %d/%d boards)' % (arch, count,
1067                               arch_count[arch]), newline=False)
1068                         printed_arch = True
1069                     Print(msg, colour=color, newline=False)
1070
1071             if printed_arch:
1072                 Print()
1073                 if show_detail:
1074                     self.PrintSizeDetail(target_list, show_bloat)
1075
1076
1077     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1078                            err_line_boards, warn_lines, warn_line_boards,
1079                            config, environment, show_sizes, show_detail,
1080                            show_bloat, show_config, show_environment):
1081         """Compare results with the base results and display delta.
1082
1083         Only boards mentioned in board_selected will be considered. This
1084         function is intended to be called repeatedly with the results of
1085         each commit. It therefore shows a 'diff' between what it saw in
1086         the last call and what it sees now.
1087
1088         Args:
1089             board_selected: Dict containing boards to summarise, keyed by
1090                 board.target
1091             board_dict: Dict containing boards for which we built this
1092                 commit, keyed by board.target. The value is an Outcome object.
1093             err_lines: A list of errors for this commit, or [] if there is
1094                 none, or we don't want to print errors
1095             err_line_boards: Dict keyed by error line, containing a list of
1096                 the Board objects with that error
1097             warn_lines: A list of warnings for this commit, or [] if there is
1098                 none, or we don't want to print errors
1099             warn_line_boards: Dict keyed by warning line, containing a list of
1100                 the Board objects with that warning
1101             config: Dictionary keyed by filename - e.g. '.config'. Each
1102                     value is itself a dictionary:
1103                         key: config name
1104                         value: config value
1105             environment: Dictionary keyed by environment variable, Each
1106                      value is the value of environment variable.
1107             show_sizes: Show image size deltas
1108             show_detail: Show size delta detail for each board if show_sizes
1109             show_bloat: Show detail for each function
1110             show_config: Show config changes
1111             show_environment: Show environment changes
1112         """
1113         def _BoardList(line, line_boards):
1114             """Helper function to get a line of boards containing a line
1115
1116             Args:
1117                 line: Error line to search for
1118                 line_boards: boards to search, each a Board
1119             Return:
1120                 List of boards with that error line, or [] if the user has not
1121                     requested such a list
1122             """
1123             boards = []
1124             board_set = set()
1125             if self._list_error_boards:
1126                 for board in line_boards[line]:
1127                     if not board in board_set:
1128                         boards.append(board)
1129                         board_set.add(board)
1130             return boards
1131
1132         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1133                             char):
1134             """Calculate the required output based on changes in errors
1135
1136             Args:
1137                 base_lines: List of errors/warnings for previous commit
1138                 base_line_boards: Dict keyed by error line, containing a list
1139                     of the Board objects with that error in the previous commit
1140                 lines: List of errors/warning for this commit, each a str
1141                 line_boards: Dict keyed by error line, containing a list
1142                     of the Board objects with that error in this commit
1143                 char: Character representing error ('') or warning ('w'). The
1144                     broken ('+') or fixed ('-') characters are added in this
1145                     function
1146
1147             Returns:
1148                 Tuple
1149                     List of ErrLine objects for 'better' lines
1150                     List of ErrLine objects for 'worse' lines
1151             """
1152             better_lines = []
1153             worse_lines = []
1154             for line in lines:
1155                 if line not in base_lines:
1156                     errline = ErrLine(char + '+', _BoardList(line, line_boards),
1157                                       line)
1158                     worse_lines.append(errline)
1159             for line in base_lines:
1160                 if line not in lines:
1161                     errline = ErrLine(char + '-',
1162                                       _BoardList(line, base_line_boards), line)
1163                     better_lines.append(errline)
1164             return better_lines, worse_lines
1165
1166         def _CalcConfig(delta, name, config):
1167             """Calculate configuration changes
1168
1169             Args:
1170                 delta: Type of the delta, e.g. '+'
1171                 name: name of the file which changed (e.g. .config)
1172                 config: configuration change dictionary
1173                     key: config name
1174                     value: config value
1175             Returns:
1176                 String containing the configuration changes which can be
1177                     printed
1178             """
1179             out = ''
1180             for key in sorted(config.keys()):
1181                 out += '%s=%s ' % (key, config[key])
1182             return '%s %s: %s' % (delta, name, out)
1183
1184         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1185             """Add changes in configuration to a list
1186
1187             Args:
1188                 lines: list to add to
1189                 name: config file name
1190                 config_plus: configurations added, dictionary
1191                     key: config name
1192                     value: config value
1193                 config_minus: configurations removed, dictionary
1194                     key: config name
1195                     value: config value
1196                 config_change: configurations changed, dictionary
1197                     key: config name
1198                     value: config value
1199             """
1200             if config_plus:
1201                 lines.append(_CalcConfig('+', name, config_plus))
1202             if config_minus:
1203                 lines.append(_CalcConfig('-', name, config_minus))
1204             if config_change:
1205                 lines.append(_CalcConfig('c', name, config_change))
1206
1207         def _OutputConfigInfo(lines):
1208             for line in lines:
1209                 if not line:
1210                     continue
1211                 if line[0] == '+':
1212                     col = self.col.GREEN
1213                 elif line[0] == '-':
1214                     col = self.col.RED
1215                 elif line[0] == 'c':
1216                     col = self.col.YELLOW
1217                 Print('   ' + line, newline=True, colour=col)
1218
1219         def _OutputErrLines(err_lines, colour):
1220             """Output the line of error/warning lines, if not empty
1221
1222             Also increments self._error_lines if err_lines not empty
1223
1224             Args:
1225                 err_lines: List of ErrLine objects, each an error or warning
1226                     line, possibly including a list of boards with that
1227                     error/warning
1228                 colour: Colour to use for output
1229             """
1230             if err_lines:
1231                 out_list = []
1232                 for line in err_lines:
1233                     boards = ''
1234                     names = [board.target for board in line.boards]
1235                     board_str = ' '.join(names) if names else ''
1236                     if board_str:
1237                         out = self.col.Color(colour, line.char + '(')
1238                         out += self.col.Color(self.col.MAGENTA, board_str,
1239                                               bright=False)
1240                         out += self.col.Color(colour, ') %s' % line.errline)
1241                     else:
1242                         out = self.col.Color(colour, line.char + line.errline)
1243                     out_list.append(out)
1244                 Print('\n'.join(out_list))
1245                 self._error_lines += 1
1246
1247
1248         ok_boards = []      # List of boards fixed since last commit
1249         warn_boards = []    # List of boards with warnings since last commit
1250         err_boards = []     # List of new broken boards since last commit
1251         new_boards = []     # List of boards that didn't exist last time
1252         unknown_boards = [] # List of boards that were not built
1253
1254         for target in board_dict:
1255             if target not in board_selected:
1256                 continue
1257
1258             # If the board was built last time, add its outcome to a list
1259             if target in self._base_board_dict:
1260                 base_outcome = self._base_board_dict[target].rc
1261                 outcome = board_dict[target]
1262                 if outcome.rc == OUTCOME_UNKNOWN:
1263                     unknown_boards.append(target)
1264                 elif outcome.rc < base_outcome:
1265                     if outcome.rc == OUTCOME_WARNING:
1266                         warn_boards.append(target)
1267                     else:
1268                         ok_boards.append(target)
1269                 elif outcome.rc > base_outcome:
1270                     if outcome.rc == OUTCOME_WARNING:
1271                         warn_boards.append(target)
1272                     else:
1273                         err_boards.append(target)
1274             else:
1275                 new_boards.append(target)
1276
1277         # Get a list of errors and warnings that have appeared, and disappeared
1278         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1279                 self._base_err_line_boards, err_lines, err_line_boards, '')
1280         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1281                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1282
1283         # Display results by arch
1284         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1285                 worse_err, better_err, worse_warn, better_warn)):
1286             arch_list = {}
1287             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1288                     self.col.GREEN)
1289             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1290                     self.col.YELLOW)
1291             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1292                     self.col.RED)
1293             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1294             if self._show_unknown:
1295                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1296                         self.col.MAGENTA)
1297             for arch, target_list in arch_list.items():
1298                 Print('%10s: %s' % (arch, target_list))
1299                 self._error_lines += 1
1300             _OutputErrLines(better_err, colour=self.col.GREEN)
1301             _OutputErrLines(worse_err, colour=self.col.RED)
1302             _OutputErrLines(better_warn, colour=self.col.CYAN)
1303             _OutputErrLines(worse_warn, colour=self.col.YELLOW)
1304
1305         if show_sizes:
1306             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1307                                   show_bloat)
1308
1309         if show_environment and self._base_environment:
1310             lines = []
1311
1312             for target in board_dict:
1313                 if target not in board_selected:
1314                     continue
1315
1316                 tbase = self._base_environment[target]
1317                 tenvironment = environment[target]
1318                 environment_plus = {}
1319                 environment_minus = {}
1320                 environment_change = {}
1321                 base = tbase.environment
1322                 for key, value in tenvironment.environment.items():
1323                     if key not in base:
1324                         environment_plus[key] = value
1325                 for key, value in base.items():
1326                     if key not in tenvironment.environment:
1327                         environment_minus[key] = value
1328                 for key, value in base.items():
1329                     new_value = tenvironment.environment.get(key)
1330                     if new_value and value != new_value:
1331                         desc = '%s -> %s' % (value, new_value)
1332                         environment_change[key] = desc
1333
1334                 _AddConfig(lines, target, environment_plus, environment_minus,
1335                            environment_change)
1336
1337             _OutputConfigInfo(lines)
1338
1339         if show_config and self._base_config:
1340             summary = {}
1341             arch_config_plus = {}
1342             arch_config_minus = {}
1343             arch_config_change = {}
1344             arch_list = []
1345
1346             for target in board_dict:
1347                 if target not in board_selected:
1348                     continue
1349                 arch = board_selected[target].arch
1350                 if arch not in arch_list:
1351                     arch_list.append(arch)
1352
1353             for arch in arch_list:
1354                 arch_config_plus[arch] = {}
1355                 arch_config_minus[arch] = {}
1356                 arch_config_change[arch] = {}
1357                 for name in self.config_filenames:
1358                     arch_config_plus[arch][name] = {}
1359                     arch_config_minus[arch][name] = {}
1360                     arch_config_change[arch][name] = {}
1361
1362             for target in board_dict:
1363                 if target not in board_selected:
1364                     continue
1365
1366                 arch = board_selected[target].arch
1367
1368                 all_config_plus = {}
1369                 all_config_minus = {}
1370                 all_config_change = {}
1371                 tbase = self._base_config[target]
1372                 tconfig = config[target]
1373                 lines = []
1374                 for name in self.config_filenames:
1375                     if not tconfig.config[name]:
1376                         continue
1377                     config_plus = {}
1378                     config_minus = {}
1379                     config_change = {}
1380                     base = tbase.config[name]
1381                     for key, value in tconfig.config[name].items():
1382                         if key not in base:
1383                             config_plus[key] = value
1384                             all_config_plus[key] = value
1385                     for key, value in base.items():
1386                         if key not in tconfig.config[name]:
1387                             config_minus[key] = value
1388                             all_config_minus[key] = value
1389                     for key, value in base.items():
1390                         new_value = tconfig.config.get(key)
1391                         if new_value and value != new_value:
1392                             desc = '%s -> %s' % (value, new_value)
1393                             config_change[key] = desc
1394                             all_config_change[key] = desc
1395
1396                     arch_config_plus[arch][name].update(config_plus)
1397                     arch_config_minus[arch][name].update(config_minus)
1398                     arch_config_change[arch][name].update(config_change)
1399
1400                     _AddConfig(lines, name, config_plus, config_minus,
1401                                config_change)
1402                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1403                            all_config_change)
1404                 summary[target] = '\n'.join(lines)
1405
1406             lines_by_target = {}
1407             for target, lines in summary.items():
1408                 if lines in lines_by_target:
1409                     lines_by_target[lines].append(target)
1410                 else:
1411                     lines_by_target[lines] = [target]
1412
1413             for arch in arch_list:
1414                 lines = []
1415                 all_plus = {}
1416                 all_minus = {}
1417                 all_change = {}
1418                 for name in self.config_filenames:
1419                     all_plus.update(arch_config_plus[arch][name])
1420                     all_minus.update(arch_config_minus[arch][name])
1421                     all_change.update(arch_config_change[arch][name])
1422                     _AddConfig(lines, name, arch_config_plus[arch][name],
1423                                arch_config_minus[arch][name],
1424                                arch_config_change[arch][name])
1425                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1426                 #arch_summary[target] = '\n'.join(lines)
1427                 if lines:
1428                     Print('%s:' % arch)
1429                     _OutputConfigInfo(lines)
1430
1431             for lines, targets in lines_by_target.items():
1432                 if not lines:
1433                     continue
1434                 Print('%s :' % ' '.join(sorted(targets)))
1435                 _OutputConfigInfo(lines.split('\n'))
1436
1437
1438         # Save our updated information for the next call to this function
1439         self._base_board_dict = board_dict
1440         self._base_err_lines = err_lines
1441         self._base_warn_lines = warn_lines
1442         self._base_err_line_boards = err_line_boards
1443         self._base_warn_line_boards = warn_line_boards
1444         self._base_config = config
1445         self._base_environment = environment
1446
1447         # Get a list of boards that did not get built, if needed
1448         not_built = []
1449         for board in board_selected:
1450             if not board in board_dict:
1451                 not_built.append(board)
1452         if not_built:
1453             Print("Boards not built (%d): %s" % (len(not_built),
1454                   ', '.join(not_built)))
1455
1456     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1457             (board_dict, err_lines, err_line_boards, warn_lines,
1458              warn_line_boards, config, environment) = self.GetResultSummary(
1459                     board_selected, commit_upto,
1460                     read_func_sizes=self._show_bloat,
1461                     read_config=self._show_config,
1462                     read_environment=self._show_environment)
1463             if commits:
1464                 msg = '%02d: %s' % (commit_upto + 1,
1465                         commits[commit_upto].subject)
1466                 Print(msg, colour=self.col.BLUE)
1467             self.PrintResultSummary(board_selected, board_dict,
1468                     err_lines if self._show_errors else [], err_line_boards,
1469                     warn_lines if self._show_errors else [], warn_line_boards,
1470                     config, environment, self._show_sizes, self._show_detail,
1471                     self._show_bloat, self._show_config, self._show_environment)
1472
1473     def ShowSummary(self, commits, board_selected):
1474         """Show a build summary for U-Boot for a given board list.
1475
1476         Reset the result summary, then repeatedly call GetResultSummary on
1477         each commit's results, then display the differences we see.
1478
1479         Args:
1480             commit: Commit objects to summarise
1481             board_selected: Dict containing boards to summarise
1482         """
1483         self.commit_count = len(commits) if commits else 1
1484         self.commits = commits
1485         self.ResetResultSummary(board_selected)
1486         self._error_lines = 0
1487
1488         for commit_upto in range(0, self.commit_count, self._step):
1489             self.ProduceResultSummary(commit_upto, commits, board_selected)
1490         if not self._error_lines:
1491             Print('(no errors to report)', colour=self.col.GREEN)
1492
1493
1494     def SetupBuild(self, board_selected, commits):
1495         """Set up ready to start a build.
1496
1497         Args:
1498             board_selected: Selected boards to build
1499             commits: Selected commits to build
1500         """
1501         # First work out how many commits we will build
1502         count = (self.commit_count + self._step - 1) // self._step
1503         self.count = len(board_selected) * count
1504         self.upto = self.warned = self.fail = 0
1505         self._timestamps = collections.deque()
1506
1507     def GetThreadDir(self, thread_num):
1508         """Get the directory path to the working dir for a thread.
1509
1510         Args:
1511             thread_num: Number of thread to check.
1512         """
1513         if self.work_in_output:
1514             return self._working_dir
1515         return os.path.join(self._working_dir, '%02d' % thread_num)
1516
1517     def _PrepareThread(self, thread_num, setup_git):
1518         """Prepare the working directory for a thread.
1519
1520         This clones or fetches the repo into the thread's work directory.
1521
1522         Args:
1523             thread_num: Thread number (0, 1, ...)
1524             setup_git: True to set up a git repo clone
1525         """
1526         thread_dir = self.GetThreadDir(thread_num)
1527         builderthread.Mkdir(thread_dir)
1528         git_dir = os.path.join(thread_dir, '.git')
1529
1530         # Clone the repo if it doesn't already exist
1531         # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1532         # we have a private index but uses the origin repo's contents?
1533         if setup_git and self.git_dir:
1534             src_dir = os.path.abspath(self.git_dir)
1535             if os.path.exists(git_dir):
1536                 Print('\rFetching repo for thread %d' % thread_num,
1537                       newline=False)
1538                 gitutil.Fetch(git_dir, thread_dir)
1539                 terminal.PrintClear()
1540             else:
1541                 Print('\rCloning repo for thread %d' % thread_num,
1542                       newline=False)
1543                 gitutil.Clone(src_dir, thread_dir)
1544                 terminal.PrintClear()
1545
1546     def _PrepareWorkingSpace(self, max_threads, setup_git):
1547         """Prepare the working directory for use.
1548
1549         Set up the git repo for each thread.
1550
1551         Args:
1552             max_threads: Maximum number of threads we expect to need.
1553             setup_git: True to set up a git repo clone
1554         """
1555         builderthread.Mkdir(self._working_dir)
1556         for thread in range(max_threads):
1557             self._PrepareThread(thread, setup_git)
1558
1559     def _GetOutputSpaceRemovals(self):
1560         """Get the output directories ready to receive files.
1561
1562         Figure out what needs to be deleted in the output directory before it
1563         can be used. We only delete old buildman directories which have the
1564         expected name pattern. See _GetOutputDir().
1565
1566         Returns:
1567             List of full paths of directories to remove
1568         """
1569         if not self.commits:
1570             return
1571         dir_list = []
1572         for commit_upto in range(self.commit_count):
1573             dir_list.append(self._GetOutputDir(commit_upto))
1574
1575         to_remove = []
1576         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1577             if dirname not in dir_list:
1578                 leaf = dirname[len(self.base_dir) + 1:]
1579                 m =  re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1580                 if m:
1581                     to_remove.append(dirname)
1582         return to_remove
1583
1584     def _PrepareOutputSpace(self):
1585         """Get the output directories ready to receive files.
1586
1587         We delete any output directories which look like ones we need to
1588         create. Having left over directories is confusing when the user wants
1589         to check the output manually.
1590         """
1591         to_remove = self._GetOutputSpaceRemovals()
1592         if to_remove:
1593             Print('Removing %d old build directories...' % len(to_remove),
1594                   newline=False)
1595             for dirname in to_remove:
1596                 shutil.rmtree(dirname)
1597             terminal.PrintClear()
1598
1599     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1600         """Build all commits for a list of boards
1601
1602         Args:
1603             commits: List of commits to be build, each a Commit object
1604             boards_selected: Dict of selected boards, key is target name,
1605                     value is Board object
1606             keep_outputs: True to save build output files
1607             verbose: Display build results as they are completed
1608         Returns:
1609             Tuple containing:
1610                 - number of boards that failed to build
1611                 - number of boards that issued warnings
1612         """
1613         self.commit_count = len(commits) if commits else 1
1614         self.commits = commits
1615         self._verbose = verbose
1616
1617         self.ResetResultSummary(board_selected)
1618         builderthread.Mkdir(self.base_dir, parents = True)
1619         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1620                 commits is not None)
1621         self._PrepareOutputSpace()
1622         Print('\rStarting build...', newline=False)
1623         self.SetupBuild(board_selected, commits)
1624         self.ProcessResult(None)
1625
1626         # Create jobs to build all commits for each board
1627         for brd in board_selected.values():
1628             job = builderthread.BuilderJob()
1629             job.board = brd
1630             job.commits = commits
1631             job.keep_outputs = keep_outputs
1632             job.work_in_output = self.work_in_output
1633             job.step = self._step
1634             self.queue.put(job)
1635
1636         term = threading.Thread(target=self.queue.join)
1637         term.setDaemon(True)
1638         term.start()
1639         while term.isAlive():
1640             term.join(100)
1641
1642         # Wait until we have processed all output
1643         self.out_queue.join()
1644         Print()
1645         return (self.fail, self.warned)