tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import Queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 import builderthread
  21 import command
  22 import gitutil
  23 import terminal
  24 from terminal import Print
  25 import toolchain
  26
  27
  28 """
  29 Theory of Operation
  30
  31 Please see README for user documentation, and you should be familiar with
  32 that before trying to make sense of this.
  33
  34 Buildman works by keeping the machine as busy as possible, building different
  35 commits for different boards on multiple CPUs at once.
  36
  37 The source repo (self.git_dir) contains all the commits to be built. Each
  38 thread works on a single board at a time. It checks out the first commit,
  39 configures it for that board, then builds it. Then it checks out the next
  40 commit and builds it (typically without re-configuring). When it runs out
  41 of commits, it gets another job from the builder and starts again with that
  42 board.
  43
  44 Clearly the builder threads could work either way - they could check out a
  45 commit and then built it for all boards. Using separate directories for each
  46 commit/board pair they could leave their build product around afterwards
  47 also.
  48
  49 The intent behind building a single board for multiple commits, is to make
  50 use of incremental builds. Since each commit is built incrementally from
  51 the previous one, builds are faster. Reconfiguring for a different board
  52 removes all intermediate object files.
  53
  54 Many threads can be working at once, but each has its own working directory.
  55 When a thread finishes a build, it puts the output files into a result
  56 directory.
  57
  58 The base directory used by buildman is normally '../<branch>', i.e.
  59 a directory higher than the source repository and named after the branch
  60 being built.
  61
  62 Within the base directory, we have one subdirectory for each commit. Within
  63 that is one subdirectory for each board. Within that is the build output for
  64 that commit/board combination.
  65
  66 Buildman also create working directories for each thread, in a .bm-work/
  67 subdirectory in the base dir.
  68
  69 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  70 'seaboard', and say that us-net has two commits. We will have directories
  71 like this:
  72
  73 us-net/             base directory
  74     01_of_02_g4ed4ebc_net--Add-tftp-speed-/
  75         sandbox/
  76             u-boot.bin
  77         seaboard/
  78             u-boot.bin
  79     02_of_02_g4ed4ebc_net--Check-tftp-comp/
  80         sandbox/
  81             u-boot.bin
  82         seaboard/
  83             u-boot.bin
  84     .bm-work/
  85         00/         working directory for thread 0 (contains source checkout)
  86             build/  build output
  87         01/         working directory for thread 1
  88             build/  build output
  89         ...
  90 u-boot/             source directory
  91     .git/           repository
  92 """
  93
  94 # Possible build outcomes
  95 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
  96
  97 # Translate a commit subject into a valid filename (and handle unicode)
  98 trans_valid_chars = string.maketrans('/: ', '---')
  99 trans_valid_chars = trans_valid_chars.decode('latin-1')
 100
 101 BASE_CONFIG_FILENAMES = [
 102     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 103 ]
 104
 105 EXTRA_CONFIG_FILENAMES = [
 106     '.config', '.config-spl', '.config-tpl',
 107     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 108     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 109 ]
 110
 111 class Config:
 112     """Holds information about configuration settings for a board."""
 113     def __init__(self, config_filename, target):
 114         self.target = target
 115         self.config = {}
 116         for fname in config_filename:
 117             self.config[fname] = {}
 118
 119     def Add(self, fname, key, value):
 120         self.config[fname][key] = value
 121
 122     def __hash__(self):
 123         val = 0
 124         for fname in self.config:
 125             for key, value in self.config[fname].iteritems():
 126                 print key, value
 127                 val = val ^ hash(key) & hash(value)
 128         return val
 129
 130 class Environment:
 131     """Holds information about environment variables for a board."""
 132     def __init__(self, target):
 133         self.target = target
 134         self.environment = {}
 135
 136     def Add(self, key, value):
 137         self.environment[key] = value
 138
 139 class Builder:
 140     """Class for building U-Boot for a particular commit.
 141
 142     Public members: (many should ->private)
 143         already_done: Number of builds already completed
 144         base_dir: Base directory to use for builder
 145         checkout: True to check out source, False to skip that step.
 146             This is used for testing.
 147         col: terminal.Color() object
 148         count: Number of commits to build
 149         do_make: Method to call to invoke Make
 150         fail: Number of builds that failed due to error
 151         force_build: Force building even if a build already exists
 152         force_config_on_failure: If a commit fails for a board, disable
 153             incremental building for the next commit we build for that
 154             board, so that we will see all warnings/errors again.
 155         force_build_failures: If a previously-built build (i.e. built on
 156             a previous run of buildman) is marked as failed, rebuild it.
 157         git_dir: Git directory containing source repository
 158         last_line_len: Length of the last line we printed (used for erasing
 159             it with new progress information)
 160         num_jobs: Number of jobs to run at once (passed to make as -j)
 161         num_threads: Number of builder threads to run
 162         out_queue: Queue of results to process
 163         re_make_err: Compiled regular expression for ignore_lines
 164         queue: Queue of jobs to run
 165         threads: List of active threads
 166         toolchains: Toolchains object to use for building
 167         upto: Current commit number we are building (0.count-1)
 168         warned: Number of builds that produced at least one warning
 169         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 170             incremental building, where buildman reconfigures on the first
 171             commit for a baord, and then just does an incremental build for
 172             the following commits. In fact buildman will reconfigure and
 173             retry for any failing commits, so generally the only effect of
 174             this option is to slow things down.
 175         in_tree: Build U-Boot in-tree instead of specifying an output
 176             directory separate from the source code. This option is really
 177             only useful for testing in-tree builds.
 178
 179     Private members:
 180         _base_board_dict: Last-summarised Dict of boards
 181         _base_err_lines: Last-summarised list of errors
 182         _base_warn_lines: Last-summarised list of warnings
 183         _build_period_us: Time taken for a single build (float object).
 184         _complete_delay: Expected delay until completion (timedelta)
 185         _next_delay_update: Next time we plan to display a progress update
 186                 (datatime)
 187         _show_unknown: Show unknown boards (those not built) in summary
 188         _timestamps: List of timestamps for the completion of the last
 189             last _timestamp_count builds. Each is a datetime object.
 190         _timestamp_count: Number of timestamps to keep in our list.
 191         _working_dir: Base working directory containing all threads
 192     """
 193     class Outcome:
 194         """Records a build outcome for a single make invocation
 195
 196         Public Members:
 197             rc: Outcome value (OUTCOME_...)
 198             err_lines: List of error lines or [] if none
 199             sizes: Dictionary of image size information, keyed by filename
 200                 - Each value is itself a dictionary containing
 201                     values for 'text', 'data' and 'bss', being the integer
 202                     size in bytes of each section.
 203             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 204                     value is itself a dictionary:
 205                         key: function name
 206                         value: Size of function in bytes
 207             config: Dictionary keyed by filename - e.g. '.config'. Each
 208                     value is itself a dictionary:
 209                         key: config name
 210                         value: config value
 211             environment: Dictionary keyed by environment variable, Each
 212                      value is the value of environment variable.
 213         """
 214         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 215                      environment):
 216             self.rc = rc
 217             self.err_lines = err_lines
 218             self.sizes = sizes
 219             self.func_sizes = func_sizes
 220             self.config = config
 221             self.environment = environment
 222
 223     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 224                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 225                  no_subdirs=False, full_path=False, verbose_build=False,
 226                  incremental=False, per_board_out_dir=False,
 227                  config_only=False, squash_config_y=False,
 228                  warnings_as_errors=False):
 229         """Create a new Builder object
 230
 231         Args:
 232             toolchains: Toolchains object to use for building
 233             base_dir: Base directory to use for builder
 234             git_dir: Git directory containing source repository
 235             num_threads: Number of builder threads to run
 236             num_jobs: Number of jobs to run at once (passed to make as -j)
 237             gnu_make: the command name of GNU Make.
 238             checkout: True to check out source, False to skip that step.
 239                 This is used for testing.
 240             show_unknown: Show unknown boards (those not built) in summary
 241             step: 1 to process every commit, n to process every nth commit
 242             no_subdirs: Don't create subdirectories when building current
 243                 source for a single board
 244             full_path: Return the full path in CROSS_COMPILE and don't set
 245                 PATH
 246             verbose_build: Run build with V=1 and don't use 'make -s'
 247             incremental: Always perform incremental builds; don't run make
 248                 mrproper when configuring
 249             per_board_out_dir: Build in a separate persistent directory per
 250                 board rather than a thread-specific directory
 251             config_only: Only configure each build, don't build it
 252             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 253             warnings_as_errors: Treat all compiler warnings as errors
 254         """
 255         self.toolchains = toolchains
 256         self.base_dir = base_dir
 257         self._working_dir = os.path.join(base_dir, '.bm-work')
 258         self.threads = []
 259         self.do_make = self.Make
 260         self.gnu_make = gnu_make
 261         self.checkout = checkout
 262         self.num_threads = num_threads
 263         self.num_jobs = num_jobs
 264         self.already_done = 0
 265         self.force_build = False
 266         self.git_dir = git_dir
 267         self._show_unknown = show_unknown
 268         self._timestamp_count = 10
 269         self._build_period_us = None
 270         self._complete_delay = None
 271         self._next_delay_update = datetime.now()
 272         self.force_config_on_failure = True
 273         self.force_build_failures = False
 274         self.force_reconfig = False
 275         self._step = step
 276         self.in_tree = False
 277         self._error_lines = 0
 278         self.no_subdirs = no_subdirs
 279         self.full_path = full_path
 280         self.verbose_build = verbose_build
 281         self.config_only = config_only
 282         self.squash_config_y = squash_config_y
 283         self.config_filenames = BASE_CONFIG_FILENAMES
 284         if not self.squash_config_y:
 285             self.config_filenames += EXTRA_CONFIG_FILENAMES
 286
 287         self.warnings_as_errors = warnings_as_errors
 288         self.col = terminal.Color()
 289
 290         self._re_function = re.compile('(.*): In function.*')
 291         self._re_files = re.compile('In file included from.*')
 292         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 293         self._re_dtb_warning = re.compile('(.*): Warning .*')
 294         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 295
 296         self.queue = Queue.Queue()
 297         self.out_queue = Queue.Queue()
 298         for i in range(self.num_threads):
 299             t = builderthread.BuilderThread(self, i, incremental,
 300                     per_board_out_dir)
 301             t.setDaemon(True)
 302             t.start()
 303             self.threads.append(t)
 304
 305         self.last_line_len = 0
 306         t = builderthread.ResultThread(self)
 307         t.setDaemon(True)
 308         t.start()
 309         self.threads.append(t)
 310
 311         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 312         self.re_make_err = re.compile('|'.join(ignore_lines))
 313
 314         # Handle existing graceful with SIGINT / Ctrl-C
 315         signal.signal(signal.SIGINT, self.signal_handler)
 316
 317     def __del__(self):
 318         """Get rid of all threads created by the builder"""
 319         for t in self.threads:
 320             del t
 321
 322     def signal_handler(self, signal, frame):
 323         sys.exit(1)
 324
 325     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 326                           show_detail=False, show_bloat=False,
 327                           list_error_boards=False, show_config=False,
 328                           show_environment=False):
 329         """Setup display options for the builder.
 330
 331         show_errors: True to show summarised error/warning info
 332         show_sizes: Show size deltas
 333         show_detail: Show detail for each board
 334         show_bloat: Show detail for each function
 335         list_error_boards: Show the boards which caused each error/warning
 336         show_config: Show config deltas
 337         show_environment: Show environment deltas
 338         """
 339         self._show_errors = show_errors
 340         self._show_sizes = show_sizes
 341         self._show_detail = show_detail
 342         self._show_bloat = show_bloat
 343         self._list_error_boards = list_error_boards
 344         self._show_config = show_config
 345         self._show_environment = show_environment
 346
 347     def _AddTimestamp(self):
 348         """Add a new timestamp to the list and record the build period.
 349
 350         The build period is the length of time taken to perform a single
 351         build (one board, one commit).
 352         """
 353         now = datetime.now()
 354         self._timestamps.append(now)
 355         count = len(self._timestamps)
 356         delta = self._timestamps[-1] - self._timestamps[0]
 357         seconds = delta.total_seconds()
 358
 359         # If we have enough data, estimate build period (time taken for a
 360         # single build) and therefore completion time.
 361         if count > 1 and self._next_delay_update < now:
 362             self._next_delay_update = now + timedelta(seconds=2)
 363             if seconds > 0:
 364                 self._build_period = float(seconds) / count
 365                 todo = self.count - self.upto
 366                 self._complete_delay = timedelta(microseconds=
 367                         self._build_period * todo * 1000000)
 368                 # Round it
 369                 self._complete_delay -= timedelta(
 370                         microseconds=self._complete_delay.microseconds)
 371
 372         if seconds > 60:
 373             self._timestamps.popleft()
 374             count -= 1
 375
 376     def ClearLine(self, length):
 377         """Clear any characters on the current line
 378
 379         Make way for a new line of length 'length', by outputting enough
 380         spaces to clear out the old line. Then remember the new length for
 381         next time.
 382
 383         Args:
 384             length: Length of new line, in characters
 385         """
 386         if length < self.last_line_len:
 387             Print(' ' * (self.last_line_len - length), newline=False)
 388             Print('\r', newline=False)
 389         self.last_line_len = length
 390         sys.stdout.flush()
 391
 392     def SelectCommit(self, commit, checkout=True):
 393         """Checkout the selected commit for this build
 394         """
 395         self.commit = commit
 396         if checkout and self.checkout:
 397             gitutil.Checkout(commit.hash)
 398
 399     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 400         """Run make
 401
 402         Args:
 403             commit: Commit object that is being built
 404             brd: Board object that is being built
 405             stage: Stage that we are at (mrproper, config, build)
 406             cwd: Directory where make should be run
 407             args: Arguments to pass to make
 408             kwargs: Arguments to pass to command.RunPipe()
 409         """
 410         cmd = [self.gnu_make] + list(args)
 411         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 412                 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
 413         if self.verbose_build:
 414             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 415             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 416         return result
 417
 418     def ProcessResult(self, result):
 419         """Process the result of a build, showing progress information
 420
 421         Args:
 422             result: A CommandResult object, which indicates the result for
 423                     a single build
 424         """
 425         col = terminal.Color()
 426         if result:
 427             target = result.brd.target
 428
 429             self.upto += 1
 430             if result.return_code != 0:
 431                 self.fail += 1
 432             elif result.stderr:
 433                 self.warned += 1
 434             if result.already_done:
 435                 self.already_done += 1
 436             if self._verbose:
 437                 Print('\r', newline=False)
 438                 self.ClearLine(0)
 439                 boards_selected = {target : result.brd}
 440                 self.ResetResultSummary(boards_selected)
 441                 self.ProduceResultSummary(result.commit_upto, self.commits,
 442                                           boards_selected)
 443         else:
 444             target = '(starting)'
 445
 446         # Display separate counts for ok, warned and fail
 447         ok = self.upto - self.warned - self.fail
 448         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 449         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 450         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 451
 452         name = ' /%-5d  ' % self.count
 453
 454         # Add our current completion time estimate
 455         self._AddTimestamp()
 456         if self._complete_delay:
 457             name += '%s  : ' % self._complete_delay
 458         # When building all boards for a commit, we can print a commit
 459         # progress message.
 460         if result and result.commit_upto is None:
 461             name += 'commit %2d/%-3d' % (self.commit_upto + 1,
 462                     self.commit_count)
 463
 464         name += target
 465         Print(line + name, newline=False)
 466         length = 16 + len(name)
 467         self.ClearLine(length)
 468
 469     def _GetOutputDir(self, commit_upto):
 470         """Get the name of the output directory for a commit number
 471
 472         The output directory is typically .../<branch>/<commit>.
 473
 474         Args:
 475             commit_upto: Commit number to use (0..self.count-1)
 476         """
 477         commit_dir = None
 478         if self.commits:
 479             commit = self.commits[commit_upto]
 480             subject = commit.subject.translate(trans_valid_chars)
 481             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
 482                     self.commit_count, commit.hash, subject[:20]))
 483         elif not self.no_subdirs:
 484             commit_dir = 'current'
 485         if not commit_dir:
 486             return self.base_dir
 487         return os.path.join(self.base_dir, commit_dir)
 488
 489     def GetBuildDir(self, commit_upto, target):
 490         """Get the name of the build directory for a commit number
 491
 492         The build directory is typically .../<branch>/<commit>/<target>.
 493
 494         Args:
 495             commit_upto: Commit number to use (0..self.count-1)
 496             target: Target name
 497         """
 498         output_dir = self._GetOutputDir(commit_upto)
 499         return os.path.join(output_dir, target)
 500
 501     def GetDoneFile(self, commit_upto, target):
 502         """Get the name of the done file for a commit number
 503
 504         Args:
 505             commit_upto: Commit number to use (0..self.count-1)
 506             target: Target name
 507         """
 508         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 509
 510     def GetSizesFile(self, commit_upto, target):
 511         """Get the name of the sizes file for a commit number
 512
 513         Args:
 514             commit_upto: Commit number to use (0..self.count-1)
 515             target: Target name
 516         """
 517         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 518
 519     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 520         """Get the name of the funcsizes file for a commit number and ELF file
 521
 522         Args:
 523             commit_upto: Commit number to use (0..self.count-1)
 524             target: Target name
 525             elf_fname: Filename of elf image
 526         """
 527         return os.path.join(self.GetBuildDir(commit_upto, target),
 528                             '%s.sizes' % elf_fname.replace('/', '-'))
 529
 530     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 531         """Get the name of the objdump file for a commit number and ELF file
 532
 533         Args:
 534             commit_upto: Commit number to use (0..self.count-1)
 535             target: Target name
 536             elf_fname: Filename of elf image
 537         """
 538         return os.path.join(self.GetBuildDir(commit_upto, target),
 539                             '%s.objdump' % elf_fname.replace('/', '-'))
 540
 541     def GetErrFile(self, commit_upto, target):
 542         """Get the name of the err file for a commit number
 543
 544         Args:
 545             commit_upto: Commit number to use (0..self.count-1)
 546             target: Target name
 547         """
 548         output_dir = self.GetBuildDir(commit_upto, target)
 549         return os.path.join(output_dir, 'err')
 550
 551     def FilterErrors(self, lines):
 552         """Filter out errors in which we have no interest
 553
 554         We should probably use map().
 555
 556         Args:
 557             lines: List of error lines, each a string
 558         Returns:
 559             New list with only interesting lines included
 560         """
 561         out_lines = []
 562         for line in lines:
 563             if not self.re_make_err.search(line):
 564                 out_lines.append(line)
 565         return out_lines
 566
 567     def ReadFuncSizes(self, fname, fd):
 568         """Read function sizes from the output of 'nm'
 569
 570         Args:
 571             fd: File containing data to read
 572             fname: Filename we are reading from (just for errors)
 573
 574         Returns:
 575             Dictionary containing size of each function in bytes, indexed by
 576             function name.
 577         """
 578         sym = {}
 579         for line in fd.readlines():
 580             try:
 581                 size, type, name = line[:-1].split()
 582             except:
 583                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 584                 continue
 585             if type in 'tTdDbB':
 586                 # function names begin with '.' on 64-bit powerpc
 587                 if '.' in name[1:]:
 588                     name = 'static.' + name.split('.')[0]
 589                 sym[name] = sym.get(name, 0) + int(size, 16)
 590         return sym
 591
 592     def _ProcessConfig(self, fname):
 593         """Read in a .config, autoconf.mk or autoconf.h file
 594
 595         This function handles all config file types. It ignores comments and
 596         any #defines which don't start with CONFIG_.
 597
 598         Args:
 599             fname: Filename to read
 600
 601         Returns:
 602             Dictionary:
 603                 key: Config name (e.g. CONFIG_DM)
 604                 value: Config value (e.g. 1)
 605         """
 606         config = {}
 607         if os.path.exists(fname):
 608             with open(fname) as fd:
 609                 for line in fd:
 610                     line = line.strip()
 611                     if line.startswith('#define'):
 612                         values = line[8:].split(' ', 1)
 613                         if len(values) > 1:
 614                             key, value = values
 615                         else:
 616                             key = values[0]
 617                             value = '1' if self.squash_config_y else ''
 618                         if not key.startswith('CONFIG_'):
 619                             continue
 620                     elif not line or line[0] in ['#', '*', '/']:
 621                         continue
 622                     else:
 623                         key, value = line.split('=', 1)
 624                     if self.squash_config_y and value == 'y':
 625                         value = '1'
 626                     config[key] = value
 627         return config
 628
 629     def _ProcessEnvironment(self, fname):
 630         """Read in a uboot.env file
 631
 632         This function reads in environment variables from a file.
 633
 634         Args:
 635             fname: Filename to read
 636
 637         Returns:
 638             Dictionary:
 639                 key: environment variable (e.g. bootlimit)
 640                 value: value of environment variable (e.g. 1)
 641         """
 642         environment = {}
 643         if os.path.exists(fname):
 644             with open(fname) as fd:
 645                 for line in fd.read().split('\0'):
 646                     try:
 647                         key, value = line.split('=', 1)
 648                         environment[key] = value
 649                     except ValueError:
 650                         # ignore lines we can't parse
 651                         pass
 652         return environment
 653
 654     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 655                         read_config, read_environment):
 656         """Work out the outcome of a build.
 657
 658         Args:
 659             commit_upto: Commit number to check (0..n-1)
 660             target: Target board to check
 661             read_func_sizes: True to read function size information
 662             read_config: True to read .config and autoconf.h files
 663             read_environment: True to read uboot.env files
 664
 665         Returns:
 666             Outcome object
 667         """
 668         done_file = self.GetDoneFile(commit_upto, target)
 669         sizes_file = self.GetSizesFile(commit_upto, target)
 670         sizes = {}
 671         func_sizes = {}
 672         config = {}
 673         environment = {}
 674         if os.path.exists(done_file):
 675             with open(done_file, 'r') as fd:
 676                 try:
 677                     return_code = int(fd.readline())
 678                 except ValueError:
 679                     # The file may be empty due to running out of disk space.
 680                     # Try a rebuild
 681                     return_code = 1
 682                 err_lines = []
 683                 err_file = self.GetErrFile(commit_upto, target)
 684                 if os.path.exists(err_file):
 685                     with open(err_file, 'r') as fd:
 686                         err_lines = self.FilterErrors(fd.readlines())
 687
 688                 # Decide whether the build was ok, failed or created warnings
 689                 if return_code:
 690                     rc = OUTCOME_ERROR
 691                 elif len(err_lines):
 692                     rc = OUTCOME_WARNING
 693                 else:
 694                     rc = OUTCOME_OK
 695
 696                 # Convert size information to our simple format
 697                 if os.path.exists(sizes_file):
 698                     with open(sizes_file, 'r') as fd:
 699                         for line in fd.readlines():
 700                             values = line.split()
 701                             rodata = 0
 702                             if len(values) > 6:
 703                                 rodata = int(values[6], 16)
 704                             size_dict = {
 705                                 'all' : int(values[0]) + int(values[1]) +
 706                                         int(values[2]),
 707                                 'text' : int(values[0]) - rodata,
 708                                 'data' : int(values[1]),
 709                                 'bss' : int(values[2]),
 710                                 'rodata' : rodata,
 711                             }
 712                             sizes[values[5]] = size_dict
 713
 714             if read_func_sizes:
 715                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 716                 for fname in glob.glob(pattern):
 717                     with open(fname, 'r') as fd:
 718                         dict_name = os.path.basename(fname).replace('.sizes',
 719                                                                     '')
 720                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 721
 722             if read_config:
 723                 output_dir = self.GetBuildDir(commit_upto, target)
 724                 for name in self.config_filenames:
 725                     fname = os.path.join(output_dir, name)
 726                     config[name] = self._ProcessConfig(fname)
 727
 728             if read_environment:
 729                 output_dir = self.GetBuildDir(commit_upto, target)
 730                 fname = os.path.join(output_dir, 'uboot.env')
 731                 environment = self._ProcessEnvironment(fname)
 732
 733             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 734                                    environment)
 735
 736         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 737
 738     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 739                          read_config, read_environment):
 740         """Calculate a summary of the results of building a commit.
 741
 742         Args:
 743             board_selected: Dict containing boards to summarise
 744             commit_upto: Commit number to summarize (0..self.count-1)
 745             read_func_sizes: True to read function size information
 746             read_config: True to read .config and autoconf.h files
 747             read_environment: True to read uboot.env files
 748
 749         Returns:
 750             Tuple:
 751                 Dict containing boards which passed building this commit.
 752                     keyed by board.target
 753                 List containing a summary of error lines
 754                 Dict keyed by error line, containing a list of the Board
 755                     objects with that error
 756                 List containing a summary of warning lines
 757                 Dict keyed by error line, containing a list of the Board
 758                     objects with that warning
 759                 Dictionary keyed by board.target. Each value is a dictionary:
 760                     key: filename - e.g. '.config'
 761                     value is itself a dictionary:
 762                         key: config name
 763                         value: config value
 764                 Dictionary keyed by board.target. Each value is a dictionary:
 765                     key: environment variable
 766                     value: value of environment variable
 767         """
 768         def AddLine(lines_summary, lines_boards, line, board):
 769             line = line.rstrip()
 770             if line in lines_boards:
 771                 lines_boards[line].append(board)
 772             else:
 773                 lines_boards[line] = [board]
 774                 lines_summary.append(line)
 775
 776         board_dict = {}
 777         err_lines_summary = []
 778         err_lines_boards = {}
 779         warn_lines_summary = []
 780         warn_lines_boards = {}
 781         config = {}
 782         environment = {}
 783
 784         for board in boards_selected.itervalues():
 785             outcome = self.GetBuildOutcome(commit_upto, board.target,
 786                                            read_func_sizes, read_config,
 787                                            read_environment)
 788             board_dict[board.target] = outcome
 789             last_func = None
 790             last_was_warning = False
 791             for line in outcome.err_lines:
 792                 if line:
 793                     if (self._re_function.match(line) or
 794                             self._re_files.match(line)):
 795                         last_func = line
 796                     else:
 797                         is_warning = (self._re_warning.match(line) or
 798                                       self._re_dtb_warning.match(line))
 799                         is_note = self._re_note.match(line)
 800                         if is_warning or (last_was_warning and is_note):
 801                             if last_func:
 802                                 AddLine(warn_lines_summary, warn_lines_boards,
 803                                         last_func, board)
 804                             AddLine(warn_lines_summary, warn_lines_boards,
 805                                     line, board)
 806                         else:
 807                             if last_func:
 808                                 AddLine(err_lines_summary, err_lines_boards,
 809                                         last_func, board)
 810                             AddLine(err_lines_summary, err_lines_boards,
 811                                     line, board)
 812                         last_was_warning = is_warning
 813                         last_func = None
 814             tconfig = Config(self.config_filenames, board.target)
 815             for fname in self.config_filenames:
 816                 if outcome.config:
 817                     for key, value in outcome.config[fname].iteritems():
 818                         tconfig.Add(fname, key, value)
 819             config[board.target] = tconfig
 820
 821             tenvironment = Environment(board.target)
 822             if outcome.environment:
 823                 for key, value in outcome.environment.iteritems():
 824                     tenvironment.Add(key, value)
 825             environment[board.target] = tenvironment
 826
 827         return (board_dict, err_lines_summary, err_lines_boards,
 828                 warn_lines_summary, warn_lines_boards, config, environment)
 829
 830     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 831         """Add an output to our list of outcomes for each architecture
 832
 833         This simple function adds failing boards (changes) to the
 834         relevant architecture string, so we can print the results out
 835         sorted by architecture.
 836
 837         Args:
 838              board_dict: Dict containing all boards
 839              arch_list: Dict keyed by arch name. Value is a string containing
 840                     a list of board names which failed for that arch.
 841              changes: List of boards to add to arch_list
 842              color: terminal.Colour object
 843         """
 844         done_arch = {}
 845         for target in changes:
 846             if target in board_dict:
 847                 arch = board_dict[target].arch
 848             else:
 849                 arch = 'unknown'
 850             str = self.col.Color(color, ' ' + target)
 851             if not arch in done_arch:
 852                 str = ' %s  %s' % (self.col.Color(color, char), str)
 853                 done_arch[arch] = True
 854             if not arch in arch_list:
 855                 arch_list[arch] = str
 856             else:
 857                 arch_list[arch] += str
 858
 859
 860     def ColourNum(self, num):
 861         color = self.col.RED if num > 0 else self.col.GREEN
 862         if num == 0:
 863             return '0'
 864         return self.col.Color(color, str(num))
 865
 866     def ResetResultSummary(self, board_selected):
 867         """Reset the results summary ready for use.
 868
 869         Set up the base board list to be all those selected, and set the
 870         error lines to empty.
 871
 872         Following this, calls to PrintResultSummary() will use this
 873         information to work out what has changed.
 874
 875         Args:
 876             board_selected: Dict containing boards to summarise, keyed by
 877                 board.target
 878         """
 879         self._base_board_dict = {}
 880         for board in board_selected:
 881             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 882                                                            {})
 883         self._base_err_lines = []
 884         self._base_warn_lines = []
 885         self._base_err_line_boards = {}
 886         self._base_warn_line_boards = {}
 887         self._base_config = None
 888         self._base_environment = None
 889
 890     def PrintFuncSizeDetail(self, fname, old, new):
 891         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 892         delta, common = [], {}
 893
 894         for a in old:
 895             if a in new:
 896                 common[a] = 1
 897
 898         for name in old:
 899             if name not in common:
 900                 remove += 1
 901                 down += old[name]
 902                 delta.append([-old[name], name])
 903
 904         for name in new:
 905             if name not in common:
 906                 add += 1
 907                 up += new[name]
 908                 delta.append([new[name], name])
 909
 910         for name in common:
 911                 diff = new.get(name, 0) - old.get(name, 0)
 912                 if diff > 0:
 913                     grow, up = grow + 1, up + diff
 914                 elif diff < 0:
 915                     shrink, down = shrink + 1, down - diff
 916                 delta.append([diff, name])
 917
 918         delta.sort()
 919         delta.reverse()
 920
 921         args = [add, -remove, grow, -shrink, up, -down, up - down]
 922         if max(args) == 0 and min(args) == 0:
 923             return
 924         args = [self.ColourNum(x) for x in args]
 925         indent = ' ' * 15
 926         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
 927               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
 928         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
 929                                          'delta'))
 930         for diff, name in delta:
 931             if diff:
 932                 color = self.col.RED if diff > 0 else self.col.GREEN
 933                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
 934                         old.get(name, '-'), new.get(name,'-'), diff)
 935                 Print(msg, colour=color)
 936
 937
 938     def PrintSizeDetail(self, target_list, show_bloat):
 939         """Show details size information for each board
 940
 941         Args:
 942             target_list: List of targets, each a dict containing:
 943                     'target': Target name
 944                     'total_diff': Total difference in bytes across all areas
 945                     <part_name>: Difference for that part
 946             show_bloat: Show detail for each function
 947         """
 948         targets_by_diff = sorted(target_list, reverse=True,
 949         key=lambda x: x['_total_diff'])
 950         for result in targets_by_diff:
 951             printed_target = False
 952             for name in sorted(result):
 953                 diff = result[name]
 954                 if name.startswith('_'):
 955                     continue
 956                 if diff != 0:
 957                     color = self.col.RED if diff > 0 else self.col.GREEN
 958                 msg = ' %s %+d' % (name, diff)
 959                 if not printed_target:
 960                     Print('%10s  %-15s:' % ('', result['_target']),
 961                           newline=False)
 962                     printed_target = True
 963                 Print(msg, colour=color, newline=False)
 964             if printed_target:
 965                 Print()
 966                 if show_bloat:
 967                     target = result['_target']
 968                     outcome = result['_outcome']
 969                     base_outcome = self._base_board_dict[target]
 970                     for fname in outcome.func_sizes:
 971                         self.PrintFuncSizeDetail(fname,
 972                                                  base_outcome.func_sizes[fname],
 973                                                  outcome.func_sizes[fname])
 974
 975
 976     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
 977                          show_bloat):
 978         """Print a summary of image sizes broken down by section.
 979
 980         The summary takes the form of one line per architecture. The
 981         line contains deltas for each of the sections (+ means the section
 982         got bigger, - means smaller). The nunmbers are the average number
 983         of bytes that a board in this section increased by.
 984
 985         For example:
 986            powerpc: (622 boards)   text -0.0
 987           arm: (285 boards)   text -0.0
 988           nds32: (3 boards)   text -8.0
 989
 990         Args:
 991             board_selected: Dict containing boards to summarise, keyed by
 992                 board.target
 993             board_dict: Dict containing boards for which we built this
 994                 commit, keyed by board.target. The value is an Outcome object.
 995             show_detail: Show detail for each board
 996             show_bloat: Show detail for each function
 997         """
 998         arch_list = {}
 999         arch_count = {}
1000
1001         # Calculate changes in size for different image parts
1002         # The previous sizes are in Board.sizes, for each board
1003         for target in board_dict:
1004             if target not in board_selected:
1005                 continue
1006             base_sizes = self._base_board_dict[target].sizes
1007             outcome = board_dict[target]
1008             sizes = outcome.sizes
1009
1010             # Loop through the list of images, creating a dict of size
1011             # changes for each image/part. We end up with something like
1012             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1013             # which means that U-Boot data increased by 5 bytes and SPL
1014             # text decreased by 4.
1015             err = {'_target' : target}
1016             for image in sizes:
1017                 if image in base_sizes:
1018                     base_image = base_sizes[image]
1019                     # Loop through the text, data, bss parts
1020                     for part in sorted(sizes[image]):
1021                         diff = sizes[image][part] - base_image[part]
1022                         col = None
1023                         if diff:
1024                             if image == 'u-boot':
1025                                 name = part
1026                             else:
1027                                 name = image + ':' + part
1028                             err[name] = diff
1029             arch = board_selected[target].arch
1030             if not arch in arch_count:
1031                 arch_count[arch] = 1
1032             else:
1033                 arch_count[arch] += 1
1034             if not sizes:
1035                 pass    # Only add to our list when we have some stats
1036             elif not arch in arch_list:
1037                 arch_list[arch] = [err]
1038             else:
1039                 arch_list[arch].append(err)
1040
1041         # We now have a list of image size changes sorted by arch
1042         # Print out a summary of these
1043         for arch, target_list in arch_list.iteritems():
1044             # Get total difference for each type
1045             totals = {}
1046             for result in target_list:
1047                 total = 0
1048                 for name, diff in result.iteritems():
1049                     if name.startswith('_'):
1050                         continue
1051                     total += diff
1052                     if name in totals:
1053                         totals[name] += diff
1054                     else:
1055                         totals[name] = diff
1056                 result['_total_diff'] = total
1057                 result['_outcome'] = board_dict[result['_target']]
1058
1059             count = len(target_list)
1060             printed_arch = False
1061             for name in sorted(totals):
1062                 diff = totals[name]
1063                 if diff:
1064                     # Display the average difference in this name for this
1065                     # architecture
1066                     avg_diff = float(diff) / count
1067                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1068                     msg = ' %s %+1.1f' % (name, avg_diff)
1069                     if not printed_arch:
1070                         Print('%10s: (for %d/%d boards)' % (arch, count,
1071                               arch_count[arch]), newline=False)
1072                         printed_arch = True
1073                     Print(msg, colour=color, newline=False)
1074
1075             if printed_arch:
1076                 Print()
1077                 if show_detail:
1078                     self.PrintSizeDetail(target_list, show_bloat)
1079
1080
1081     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1082                            err_line_boards, warn_lines, warn_line_boards,
1083                            config, environment, show_sizes, show_detail,
1084                            show_bloat, show_config, show_environment):
1085         """Compare results with the base results and display delta.
1086
1087         Only boards mentioned in board_selected will be considered. This
1088         function is intended to be called repeatedly with the results of
1089         each commit. It therefore shows a 'diff' between what it saw in
1090         the last call and what it sees now.
1091
1092         Args:
1093             board_selected: Dict containing boards to summarise, keyed by
1094                 board.target
1095             board_dict: Dict containing boards for which we built this
1096                 commit, keyed by board.target. The value is an Outcome object.
1097             err_lines: A list of errors for this commit, or [] if there is
1098                 none, or we don't want to print errors
1099             err_line_boards: Dict keyed by error line, containing a list of
1100                 the Board objects with that error
1101             warn_lines: A list of warnings for this commit, or [] if there is
1102                 none, or we don't want to print errors
1103             warn_line_boards: Dict keyed by warning line, containing a list of
1104                 the Board objects with that warning
1105             config: Dictionary keyed by filename - e.g. '.config'. Each
1106                     value is itself a dictionary:
1107                         key: config name
1108                         value: config value
1109             environment: Dictionary keyed by environment variable, Each
1110                      value is the value of environment variable.
1111             show_sizes: Show image size deltas
1112             show_detail: Show detail for each board
1113             show_bloat: Show detail for each function
1114             show_config: Show config changes
1115             show_environment: Show environment changes
1116         """
1117         def _BoardList(line, line_boards):
1118             """Helper function to get a line of boards containing a line
1119
1120             Args:
1121                 line: Error line to search for
1122             Return:
1123                 String containing a list of boards with that error line, or
1124                 '' if the user has not requested such a list
1125             """
1126             if self._list_error_boards:
1127                 names = []
1128                 for board in line_boards[line]:
1129                     if not board.target in names:
1130                         names.append(board.target)
1131                 names_str = '(%s) ' % ','.join(names)
1132             else:
1133                 names_str = ''
1134             return names_str
1135
1136         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1137                             char):
1138             better_lines = []
1139             worse_lines = []
1140             for line in lines:
1141                 if line not in base_lines:
1142                     worse_lines.append(char + '+' +
1143                             _BoardList(line, line_boards) + line)
1144             for line in base_lines:
1145                 if line not in lines:
1146                     better_lines.append(char + '-' +
1147                             _BoardList(line, base_line_boards) + line)
1148             return better_lines, worse_lines
1149
1150         def _CalcConfig(delta, name, config):
1151             """Calculate configuration changes
1152
1153             Args:
1154                 delta: Type of the delta, e.g. '+'
1155                 name: name of the file which changed (e.g. .config)
1156                 config: configuration change dictionary
1157                     key: config name
1158                     value: config value
1159             Returns:
1160                 String containing the configuration changes which can be
1161                     printed
1162             """
1163             out = ''
1164             for key in sorted(config.keys()):
1165                 out += '%s=%s ' % (key, config[key])
1166             return '%s %s: %s' % (delta, name, out)
1167
1168         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1169             """Add changes in configuration to a list
1170
1171             Args:
1172                 lines: list to add to
1173                 name: config file name
1174                 config_plus: configurations added, dictionary
1175                     key: config name
1176                     value: config value
1177                 config_minus: configurations removed, dictionary
1178                     key: config name
1179                     value: config value
1180                 config_change: configurations changed, dictionary
1181                     key: config name
1182                     value: config value
1183             """
1184             if config_plus:
1185                 lines.append(_CalcConfig('+', name, config_plus))
1186             if config_minus:
1187                 lines.append(_CalcConfig('-', name, config_minus))
1188             if config_change:
1189                 lines.append(_CalcConfig('c', name, config_change))
1190
1191         def _OutputConfigInfo(lines):
1192             for line in lines:
1193                 if not line:
1194                     continue
1195                 if line[0] == '+':
1196                     col = self.col.GREEN
1197                 elif line[0] == '-':
1198                     col = self.col.RED
1199                 elif line[0] == 'c':
1200                     col = self.col.YELLOW
1201                 Print('   ' + line, newline=True, colour=col)
1202
1203
1204         ok_boards = []      # List of boards fixed since last commit
1205         warn_boards = []    # List of boards with warnings since last commit
1206         err_boards = []     # List of new broken boards since last commit
1207         new_boards = []     # List of boards that didn't exist last time
1208         unknown_boards = [] # List of boards that were not built
1209
1210         for target in board_dict:
1211             if target not in board_selected:
1212                 continue
1213
1214             # If the board was built last time, add its outcome to a list
1215             if target in self._base_board_dict:
1216                 base_outcome = self._base_board_dict[target].rc
1217                 outcome = board_dict[target]
1218                 if outcome.rc == OUTCOME_UNKNOWN:
1219                     unknown_boards.append(target)
1220                 elif outcome.rc < base_outcome:
1221                     if outcome.rc == OUTCOME_WARNING:
1222                         warn_boards.append(target)
1223                     else:
1224                         ok_boards.append(target)
1225                 elif outcome.rc > base_outcome:
1226                     if outcome.rc == OUTCOME_WARNING:
1227                         warn_boards.append(target)
1228                     else:
1229                         err_boards.append(target)
1230             else:
1231                 new_boards.append(target)
1232
1233         # Get a list of errors that have appeared, and disappeared
1234         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1235                 self._base_err_line_boards, err_lines, err_line_boards, '')
1236         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1237                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1238
1239         # Display results by arch
1240         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1241                 worse_err, better_err, worse_warn, better_warn)):
1242             arch_list = {}
1243             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1244                     self.col.GREEN)
1245             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1246                     self.col.YELLOW)
1247             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1248                     self.col.RED)
1249             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1250             if self._show_unknown:
1251                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1252                         self.col.MAGENTA)
1253             for arch, target_list in arch_list.iteritems():
1254                 Print('%10s: %s' % (arch, target_list))
1255                 self._error_lines += 1
1256             if better_err:
1257                 Print('\n'.join(better_err), colour=self.col.GREEN)
1258                 self._error_lines += 1
1259             if worse_err:
1260                 Print('\n'.join(worse_err), colour=self.col.RED)
1261                 self._error_lines += 1
1262             if better_warn:
1263                 Print('\n'.join(better_warn), colour=self.col.CYAN)
1264                 self._error_lines += 1
1265             if worse_warn:
1266                 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
1267                 self._error_lines += 1
1268
1269         if show_sizes:
1270             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1271                                   show_bloat)
1272
1273         if show_environment and self._base_environment:
1274             lines = []
1275
1276             for target in board_dict:
1277                 if target not in board_selected:
1278                     continue
1279
1280                 tbase = self._base_environment[target]
1281                 tenvironment = environment[target]
1282                 environment_plus = {}
1283                 environment_minus = {}
1284                 environment_change = {}
1285                 base = tbase.environment
1286                 for key, value in tenvironment.environment.iteritems():
1287                     if key not in base:
1288                         environment_plus[key] = value
1289                 for key, value in base.iteritems():
1290                     if key not in tenvironment.environment:
1291                         environment_minus[key] = value
1292                 for key, value in base.iteritems():
1293                     new_value = tenvironment.environment.get(key)
1294                     if new_value and value != new_value:
1295                         desc = '%s -> %s' % (value, new_value)
1296                         environment_change[key] = desc
1297
1298                 _AddConfig(lines, target, environment_plus, environment_minus,
1299                            environment_change)
1300
1301             _OutputConfigInfo(lines)
1302
1303         if show_config and self._base_config:
1304             summary = {}
1305             arch_config_plus = {}
1306             arch_config_minus = {}
1307             arch_config_change = {}
1308             arch_list = []
1309
1310             for target in board_dict:
1311                 if target not in board_selected:
1312                     continue
1313                 arch = board_selected[target].arch
1314                 if arch not in arch_list:
1315                     arch_list.append(arch)
1316
1317             for arch in arch_list:
1318                 arch_config_plus[arch] = {}
1319                 arch_config_minus[arch] = {}
1320                 arch_config_change[arch] = {}
1321                 for name in self.config_filenames:
1322                     arch_config_plus[arch][name] = {}
1323                     arch_config_minus[arch][name] = {}
1324                     arch_config_change[arch][name] = {}
1325
1326             for target in board_dict:
1327                 if target not in board_selected:
1328                     continue
1329
1330                 arch = board_selected[target].arch
1331
1332                 all_config_plus = {}
1333                 all_config_minus = {}
1334                 all_config_change = {}
1335                 tbase = self._base_config[target]
1336                 tconfig = config[target]
1337                 lines = []
1338                 for name in self.config_filenames:
1339                     if not tconfig.config[name]:
1340                         continue
1341                     config_plus = {}
1342                     config_minus = {}
1343                     config_change = {}
1344                     base = tbase.config[name]
1345                     for key, value in tconfig.config[name].iteritems():
1346                         if key not in base:
1347                             config_plus[key] = value
1348                             all_config_plus[key] = value
1349                     for key, value in base.iteritems():
1350                         if key not in tconfig.config[name]:
1351                             config_minus[key] = value
1352                             all_config_minus[key] = value
1353                     for key, value in base.iteritems():
1354                         new_value = tconfig.config.get(key)
1355                         if new_value and value != new_value:
1356                             desc = '%s -> %s' % (value, new_value)
1357                             config_change[key] = desc
1358                             all_config_change[key] = desc
1359
1360                     arch_config_plus[arch][name].update(config_plus)
1361                     arch_config_minus[arch][name].update(config_minus)
1362                     arch_config_change[arch][name].update(config_change)
1363
1364                     _AddConfig(lines, name, config_plus, config_minus,
1365                                config_change)
1366                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1367                            all_config_change)
1368                 summary[target] = '\n'.join(lines)
1369
1370             lines_by_target = {}
1371             for target, lines in summary.iteritems():
1372                 if lines in lines_by_target:
1373                     lines_by_target[lines].append(target)
1374                 else:
1375                     lines_by_target[lines] = [target]
1376
1377             for arch in arch_list:
1378                 lines = []
1379                 all_plus = {}
1380                 all_minus = {}
1381                 all_change = {}
1382                 for name in self.config_filenames:
1383                     all_plus.update(arch_config_plus[arch][name])
1384                     all_minus.update(arch_config_minus[arch][name])
1385                     all_change.update(arch_config_change[arch][name])
1386                     _AddConfig(lines, name, arch_config_plus[arch][name],
1387                                arch_config_minus[arch][name],
1388                                arch_config_change[arch][name])
1389                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1390                 #arch_summary[target] = '\n'.join(lines)
1391                 if lines:
1392                     Print('%s:' % arch)
1393                     _OutputConfigInfo(lines)
1394
1395             for lines, targets in lines_by_target.iteritems():
1396                 if not lines:
1397                     continue
1398                 Print('%s :' % ' '.join(sorted(targets)))
1399                 _OutputConfigInfo(lines.split('\n'))
1400
1401
1402         # Save our updated information for the next call to this function
1403         self._base_board_dict = board_dict
1404         self._base_err_lines = err_lines
1405         self._base_warn_lines = warn_lines
1406         self._base_err_line_boards = err_line_boards
1407         self._base_warn_line_boards = warn_line_boards
1408         self._base_config = config
1409         self._base_environment = environment
1410
1411         # Get a list of boards that did not get built, if needed
1412         not_built = []
1413         for board in board_selected:
1414             if not board in board_dict:
1415                 not_built.append(board)
1416         if not_built:
1417             Print("Boards not built (%d): %s" % (len(not_built),
1418                   ', '.join(not_built)))
1419
1420     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1421             (board_dict, err_lines, err_line_boards, warn_lines,
1422              warn_line_boards, config, environment) = self.GetResultSummary(
1423                     board_selected, commit_upto,
1424                     read_func_sizes=self._show_bloat,
1425                     read_config=self._show_config,
1426                     read_environment=self._show_environment)
1427             if commits:
1428                 msg = '%02d: %s' % (commit_upto + 1,
1429                         commits[commit_upto].subject)
1430                 Print(msg, colour=self.col.BLUE)
1431             self.PrintResultSummary(board_selected, board_dict,
1432                     err_lines if self._show_errors else [], err_line_boards,
1433                     warn_lines if self._show_errors else [], warn_line_boards,
1434                     config, environment, self._show_sizes, self._show_detail,
1435                     self._show_bloat, self._show_config, self._show_environment)
1436
1437     def ShowSummary(self, commits, board_selected):
1438         """Show a build summary for U-Boot for a given board list.
1439
1440         Reset the result summary, then repeatedly call GetResultSummary on
1441         each commit's results, then display the differences we see.
1442
1443         Args:
1444             commit: Commit objects to summarise
1445             board_selected: Dict containing boards to summarise
1446         """
1447         self.commit_count = len(commits) if commits else 1
1448         self.commits = commits
1449         self.ResetResultSummary(board_selected)
1450         self._error_lines = 0
1451
1452         for commit_upto in range(0, self.commit_count, self._step):
1453             self.ProduceResultSummary(commit_upto, commits, board_selected)
1454         if not self._error_lines:
1455             Print('(no errors to report)', colour=self.col.GREEN)
1456
1457
1458     def SetupBuild(self, board_selected, commits):
1459         """Set up ready to start a build.
1460
1461         Args:
1462             board_selected: Selected boards to build
1463             commits: Selected commits to build
1464         """
1465         # First work out how many commits we will build
1466         count = (self.commit_count + self._step - 1) / self._step
1467         self.count = len(board_selected) * count
1468         self.upto = self.warned = self.fail = 0
1469         self._timestamps = collections.deque()
1470
1471     def GetThreadDir(self, thread_num):
1472         """Get the directory path to the working dir for a thread.
1473
1474         Args:
1475             thread_num: Number of thread to check.
1476         """
1477         return os.path.join(self._working_dir, '%02d' % thread_num)
1478
1479     def _PrepareThread(self, thread_num, setup_git):
1480         """Prepare the working directory for a thread.
1481
1482         This clones or fetches the repo into the thread's work directory.
1483
1484         Args:
1485             thread_num: Thread number (0, 1, ...)
1486             setup_git: True to set up a git repo clone
1487         """
1488         thread_dir = self.GetThreadDir(thread_num)
1489         builderthread.Mkdir(thread_dir)
1490         git_dir = os.path.join(thread_dir, '.git')
1491
1492         # Clone the repo if it doesn't already exist
1493         # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1494         # we have a private index but uses the origin repo's contents?
1495         if setup_git and self.git_dir:
1496             src_dir = os.path.abspath(self.git_dir)
1497             if os.path.exists(git_dir):
1498                 gitutil.Fetch(git_dir, thread_dir)
1499             else:
1500                 Print('\rCloning repo for thread %d' % thread_num,
1501                       newline=False)
1502                 gitutil.Clone(src_dir, thread_dir)
1503                 Print('\r%s\r' % (' ' * 30), newline=False)
1504
1505     def _PrepareWorkingSpace(self, max_threads, setup_git):
1506         """Prepare the working directory for use.
1507
1508         Set up the git repo for each thread.
1509
1510         Args:
1511             max_threads: Maximum number of threads we expect to need.
1512             setup_git: True to set up a git repo clone
1513         """
1514         builderthread.Mkdir(self._working_dir)
1515         for thread in range(max_threads):
1516             self._PrepareThread(thread, setup_git)
1517
1518     def _PrepareOutputSpace(self):
1519         """Get the output directories ready to receive files.
1520
1521         We delete any output directories which look like ones we need to
1522         create. Having left over directories is confusing when the user wants
1523         to check the output manually.
1524         """
1525         if not self.commits:
1526             return
1527         dir_list = []
1528         for commit_upto in range(self.commit_count):
1529             dir_list.append(self._GetOutputDir(commit_upto))
1530
1531         to_remove = []
1532         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1533             if dirname not in dir_list:
1534                 to_remove.append(dirname)
1535         if to_remove:
1536             Print('Removing %d old build directories' % len(to_remove),
1537                   newline=False)
1538             for dirname in to_remove:
1539                 shutil.rmtree(dirname)
1540
1541     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1542         """Build all commits for a list of boards
1543
1544         Args:
1545             commits: List of commits to be build, each a Commit object
1546             boards_selected: Dict of selected boards, key is target name,
1547                     value is Board object
1548             keep_outputs: True to save build output files
1549             verbose: Display build results as they are completed
1550         Returns:
1551             Tuple containing:
1552                 - number of boards that failed to build
1553                 - number of boards that issued warnings
1554         """
1555         self.commit_count = len(commits) if commits else 1
1556         self.commits = commits
1557         self._verbose = verbose
1558
1559         self.ResetResultSummary(board_selected)
1560         builderthread.Mkdir(self.base_dir, parents = True)
1561         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1562                 commits is not None)
1563         self._PrepareOutputSpace()
1564         Print('\rStarting build...', newline=False)
1565         self.SetupBuild(board_selected, commits)
1566         self.ProcessResult(None)
1567
1568         # Create jobs to build all commits for each board
1569         for brd in board_selected.itervalues():
1570             job = builderthread.BuilderJob()
1571             job.board = brd
1572             job.commits = commits
1573             job.keep_outputs = keep_outputs
1574             job.step = self._step
1575             self.queue.put(job)
1576
1577         term = threading.Thread(target=self.queue.join)
1578         term.setDaemon(True)
1579         term.start()
1580         while term.isAlive():
1581             term.join(100)
1582
1583         # Wait until we have processed all output
1584         self.out_queue.join()
1585         Print()
1586         self.ClearLine(0)
1587         return (self.fail, self.warned)