tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 import builderthread
  21 import command
  22 import gitutil
  23 import terminal
  24 from terminal import Print
  25 import toolchain
  26
  27
  28 """
  29 Theory of Operation
  30
  31 Please see README for user documentation, and you should be familiar with
  32 that before trying to make sense of this.
  33
  34 Buildman works by keeping the machine as busy as possible, building different
  35 commits for different boards on multiple CPUs at once.
  36
  37 The source repo (self.git_dir) contains all the commits to be built. Each
  38 thread works on a single board at a time. It checks out the first commit,
  39 configures it for that board, then builds it. Then it checks out the next
  40 commit and builds it (typically without re-configuring). When it runs out
  41 of commits, it gets another job from the builder and starts again with that
  42 board.
  43
  44 Clearly the builder threads could work either way - they could check out a
  45 commit and then built it for all boards. Using separate directories for each
  46 commit/board pair they could leave their build product around afterwards
  47 also.
  48
  49 The intent behind building a single board for multiple commits, is to make
  50 use of incremental builds. Since each commit is built incrementally from
  51 the previous one, builds are faster. Reconfiguring for a different board
  52 removes all intermediate object files.
  53
  54 Many threads can be working at once, but each has its own working directory.
  55 When a thread finishes a build, it puts the output files into a result
  56 directory.
  57
  58 The base directory used by buildman is normally '../<branch>', i.e.
  59 a directory higher than the source repository and named after the branch
  60 being built.
  61
  62 Within the base directory, we have one subdirectory for each commit. Within
  63 that is one subdirectory for each board. Within that is the build output for
  64 that commit/board combination.
  65
  66 Buildman also create working directories for each thread, in a .bm-work/
  67 subdirectory in the base dir.
  68
  69 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  70 'seaboard', and say that us-net has two commits. We will have directories
  71 like this:
  72
  73 us-net/             base directory
  74     01_of_02_g4ed4ebc_net--Add-tftp-speed-/
  75         sandbox/
  76             u-boot.bin
  77         seaboard/
  78             u-boot.bin
  79     02_of_02_g4ed4ebc_net--Check-tftp-comp/
  80         sandbox/
  81             u-boot.bin
  82         seaboard/
  83             u-boot.bin
  84     .bm-work/
  85         00/         working directory for thread 0 (contains source checkout)
  86             build/  build output
  87         01/         working directory for thread 1
  88             build/  build output
  89         ...
  90 u-boot/             source directory
  91     .git/           repository
  92 """
  93
  94 # Possible build outcomes
  95 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
  96
  97 # Translate a commit subject into a valid filename (and handle unicode)
  98 trans_valid_chars = str.maketrans('/: ', '---')
  99
 100 BASE_CONFIG_FILENAMES = [
 101     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 102 ]
 103
 104 EXTRA_CONFIG_FILENAMES = [
 105     '.config', '.config-spl', '.config-tpl',
 106     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 107     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 108 ]
 109
 110 class Config:
 111     """Holds information about configuration settings for a board."""
 112     def __init__(self, config_filename, target):
 113         self.target = target
 114         self.config = {}
 115         for fname in config_filename:
 116             self.config[fname] = {}
 117
 118     def Add(self, fname, key, value):
 119         self.config[fname][key] = value
 120
 121     def __hash__(self):
 122         val = 0
 123         for fname in self.config:
 124             for key, value in self.config[fname].items():
 125                 print(key, value)
 126                 val = val ^ hash(key) & hash(value)
 127         return val
 128
 129 class Environment:
 130     """Holds information about environment variables for a board."""
 131     def __init__(self, target):
 132         self.target = target
 133         self.environment = {}
 134
 135     def Add(self, key, value):
 136         self.environment[key] = value
 137
 138 class Builder:
 139     """Class for building U-Boot for a particular commit.
 140
 141     Public members: (many should ->private)
 142         already_done: Number of builds already completed
 143         base_dir: Base directory to use for builder
 144         checkout: True to check out source, False to skip that step.
 145             This is used for testing.
 146         col: terminal.Color() object
 147         count: Number of commits to build
 148         do_make: Method to call to invoke Make
 149         fail: Number of builds that failed due to error
 150         force_build: Force building even if a build already exists
 151         force_config_on_failure: If a commit fails for a board, disable
 152             incremental building for the next commit we build for that
 153             board, so that we will see all warnings/errors again.
 154         force_build_failures: If a previously-built build (i.e. built on
 155             a previous run of buildman) is marked as failed, rebuild it.
 156         git_dir: Git directory containing source repository
 157         last_line_len: Length of the last line we printed (used for erasing
 158             it with new progress information)
 159         num_jobs: Number of jobs to run at once (passed to make as -j)
 160         num_threads: Number of builder threads to run
 161         out_queue: Queue of results to process
 162         re_make_err: Compiled regular expression for ignore_lines
 163         queue: Queue of jobs to run
 164         threads: List of active threads
 165         toolchains: Toolchains object to use for building
 166         upto: Current commit number we are building (0.count-1)
 167         warned: Number of builds that produced at least one warning
 168         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 169             incremental building, where buildman reconfigures on the first
 170             commit for a baord, and then just does an incremental build for
 171             the following commits. In fact buildman will reconfigure and
 172             retry for any failing commits, so generally the only effect of
 173             this option is to slow things down.
 174         in_tree: Build U-Boot in-tree instead of specifying an output
 175             directory separate from the source code. This option is really
 176             only useful for testing in-tree builds.
 177         work_in_output: Use the output directory as the work directory and
 178             don't write to a separate output directory.
 179
 180     Private members:
 181         _base_board_dict: Last-summarised Dict of boards
 182         _base_err_lines: Last-summarised list of errors
 183         _base_warn_lines: Last-summarised list of warnings
 184         _build_period_us: Time taken for a single build (float object).
 185         _complete_delay: Expected delay until completion (timedelta)
 186         _next_delay_update: Next time we plan to display a progress update
 187                 (datatime)
 188         _show_unknown: Show unknown boards (those not built) in summary
 189         _timestamps: List of timestamps for the completion of the last
 190             last _timestamp_count builds. Each is a datetime object.
 191         _timestamp_count: Number of timestamps to keep in our list.
 192         _working_dir: Base working directory containing all threads
 193     """
 194     class Outcome:
 195         """Records a build outcome for a single make invocation
 196
 197         Public Members:
 198             rc: Outcome value (OUTCOME_...)
 199             err_lines: List of error lines or [] if none
 200             sizes: Dictionary of image size information, keyed by filename
 201                 - Each value is itself a dictionary containing
 202                     values for 'text', 'data' and 'bss', being the integer
 203                     size in bytes of each section.
 204             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 205                     value is itself a dictionary:
 206                         key: function name
 207                         value: Size of function in bytes
 208             config: Dictionary keyed by filename - e.g. '.config'. Each
 209                     value is itself a dictionary:
 210                         key: config name
 211                         value: config value
 212             environment: Dictionary keyed by environment variable, Each
 213                      value is the value of environment variable.
 214         """
 215         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 216                      environment):
 217             self.rc = rc
 218             self.err_lines = err_lines
 219             self.sizes = sizes
 220             self.func_sizes = func_sizes
 221             self.config = config
 222             self.environment = environment
 223
 224     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 225                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 226                  no_subdirs=False, full_path=False, verbose_build=False,
 227                  incremental=False, per_board_out_dir=False,
 228                  config_only=False, squash_config_y=False,
 229                  warnings_as_errors=False, work_in_output=False):
 230         """Create a new Builder object
 231
 232         Args:
 233             toolchains: Toolchains object to use for building
 234             base_dir: Base directory to use for builder
 235             git_dir: Git directory containing source repository
 236             num_threads: Number of builder threads to run
 237             num_jobs: Number of jobs to run at once (passed to make as -j)
 238             gnu_make: the command name of GNU Make.
 239             checkout: True to check out source, False to skip that step.
 240                 This is used for testing.
 241             show_unknown: Show unknown boards (those not built) in summary
 242             step: 1 to process every commit, n to process every nth commit
 243             no_subdirs: Don't create subdirectories when building current
 244                 source for a single board
 245             full_path: Return the full path in CROSS_COMPILE and don't set
 246                 PATH
 247             verbose_build: Run build with V=1 and don't use 'make -s'
 248             incremental: Always perform incremental builds; don't run make
 249                 mrproper when configuring
 250             per_board_out_dir: Build in a separate persistent directory per
 251                 board rather than a thread-specific directory
 252             config_only: Only configure each build, don't build it
 253             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 254             warnings_as_errors: Treat all compiler warnings as errors
 255             work_in_output: Use the output directory as the work directory and
 256                 don't write to a separate output directory.
 257         """
 258         self.toolchains = toolchains
 259         self.base_dir = base_dir
 260         if work_in_output:
 261             self._working_dir = base_dir
 262         else:
 263             self._working_dir = os.path.join(base_dir, '.bm-work')
 264         self.threads = []
 265         self.do_make = self.Make
 266         self.gnu_make = gnu_make
 267         self.checkout = checkout
 268         self.num_threads = num_threads
 269         self.num_jobs = num_jobs
 270         self.already_done = 0
 271         self.force_build = False
 272         self.git_dir = git_dir
 273         self._show_unknown = show_unknown
 274         self._timestamp_count = 10
 275         self._build_period_us = None
 276         self._complete_delay = None
 277         self._next_delay_update = datetime.now()
 278         self.force_config_on_failure = True
 279         self.force_build_failures = False
 280         self.force_reconfig = False
 281         self._step = step
 282         self.in_tree = False
 283         self._error_lines = 0
 284         self.no_subdirs = no_subdirs
 285         self.full_path = full_path
 286         self.verbose_build = verbose_build
 287         self.config_only = config_only
 288         self.squash_config_y = squash_config_y
 289         self.config_filenames = BASE_CONFIG_FILENAMES
 290         self.work_in_output = work_in_output
 291         if not self.squash_config_y:
 292             self.config_filenames += EXTRA_CONFIG_FILENAMES
 293
 294         self.warnings_as_errors = warnings_as_errors
 295         self.col = terminal.Color()
 296
 297         self._re_function = re.compile('(.*): In function.*')
 298         self._re_files = re.compile('In file included from.*')
 299         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 300         self._re_dtb_warning = re.compile('(.*): Warning .*')
 301         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 302
 303         self.queue = queue.Queue()
 304         self.out_queue = queue.Queue()
 305         for i in range(self.num_threads):
 306             t = builderthread.BuilderThread(self, i, incremental,
 307                     per_board_out_dir)
 308             t.setDaemon(True)
 309             t.start()
 310             self.threads.append(t)
 311
 312         self.last_line_len = 0
 313         t = builderthread.ResultThread(self)
 314         t.setDaemon(True)
 315         t.start()
 316         self.threads.append(t)
 317
 318         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 319         self.re_make_err = re.compile('|'.join(ignore_lines))
 320
 321         # Handle existing graceful with SIGINT / Ctrl-C
 322         signal.signal(signal.SIGINT, self.signal_handler)
 323
 324     def __del__(self):
 325         """Get rid of all threads created by the builder"""
 326         for t in self.threads:
 327             del t
 328
 329     def signal_handler(self, signal, frame):
 330         sys.exit(1)
 331
 332     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 333                           show_detail=False, show_bloat=False,
 334                           list_error_boards=False, show_config=False,
 335                           show_environment=False):
 336         """Setup display options for the builder.
 337
 338         show_errors: True to show summarised error/warning info
 339         show_sizes: Show size deltas
 340         show_detail: Show size delta detail for each board if show_sizes
 341         show_bloat: Show detail for each function
 342         list_error_boards: Show the boards which caused each error/warning
 343         show_config: Show config deltas
 344         show_environment: Show environment deltas
 345         """
 346         self._show_errors = show_errors
 347         self._show_sizes = show_sizes
 348         self._show_detail = show_detail
 349         self._show_bloat = show_bloat
 350         self._list_error_boards = list_error_boards
 351         self._show_config = show_config
 352         self._show_environment = show_environment
 353
 354     def _AddTimestamp(self):
 355         """Add a new timestamp to the list and record the build period.
 356
 357         The build period is the length of time taken to perform a single
 358         build (one board, one commit).
 359         """
 360         now = datetime.now()
 361         self._timestamps.append(now)
 362         count = len(self._timestamps)
 363         delta = self._timestamps[-1] - self._timestamps[0]
 364         seconds = delta.total_seconds()
 365
 366         # If we have enough data, estimate build period (time taken for a
 367         # single build) and therefore completion time.
 368         if count > 1 and self._next_delay_update < now:
 369             self._next_delay_update = now + timedelta(seconds=2)
 370             if seconds > 0:
 371                 self._build_period = float(seconds) / count
 372                 todo = self.count - self.upto
 373                 self._complete_delay = timedelta(microseconds=
 374                         self._build_period * todo * 1000000)
 375                 # Round it
 376                 self._complete_delay -= timedelta(
 377                         microseconds=self._complete_delay.microseconds)
 378
 379         if seconds > 60:
 380             self._timestamps.popleft()
 381             count -= 1
 382
 383     def ClearLine(self, length):
 384         """Clear any characters on the current line
 385
 386         Make way for a new line of length 'length', by outputting enough
 387         spaces to clear out the old line. Then remember the new length for
 388         next time.
 389
 390         Args:
 391             length: Length of new line, in characters
 392         """
 393         if length < self.last_line_len:
 394             Print(' ' * (self.last_line_len - length), newline=False)
 395             Print('\r', newline=False)
 396         self.last_line_len = length
 397         sys.stdout.flush()
 398
 399     def SelectCommit(self, commit, checkout=True):
 400         """Checkout the selected commit for this build
 401         """
 402         self.commit = commit
 403         if checkout and self.checkout:
 404             gitutil.Checkout(commit.hash)
 405
 406     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 407         """Run make
 408
 409         Args:
 410             commit: Commit object that is being built
 411             brd: Board object that is being built
 412             stage: Stage that we are at (mrproper, config, build)
 413             cwd: Directory where make should be run
 414             args: Arguments to pass to make
 415             kwargs: Arguments to pass to command.RunPipe()
 416         """
 417         cmd = [self.gnu_make] + list(args)
 418         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 419                 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
 420         if self.verbose_build:
 421             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 422             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 423         return result
 424
 425     def ProcessResult(self, result):
 426         """Process the result of a build, showing progress information
 427
 428         Args:
 429             result: A CommandResult object, which indicates the result for
 430                     a single build
 431         """
 432         col = terminal.Color()
 433         if result:
 434             target = result.brd.target
 435
 436             self.upto += 1
 437             if result.return_code != 0:
 438                 self.fail += 1
 439             elif result.stderr:
 440                 self.warned += 1
 441             if result.already_done:
 442                 self.already_done += 1
 443             if self._verbose:
 444                 Print('\r', newline=False)
 445                 self.ClearLine(0)
 446                 boards_selected = {target : result.brd}
 447                 self.ResetResultSummary(boards_selected)
 448                 self.ProduceResultSummary(result.commit_upto, self.commits,
 449                                           boards_selected)
 450         else:
 451             target = '(starting)'
 452
 453         # Display separate counts for ok, warned and fail
 454         ok = self.upto - self.warned - self.fail
 455         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 456         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 457         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 458
 459         name = ' /%-5d  ' % self.count
 460
 461         # Add our current completion time estimate
 462         self._AddTimestamp()
 463         if self._complete_delay:
 464             name += '%s  : ' % self._complete_delay
 465         # When building all boards for a commit, we can print a commit
 466         # progress message.
 467         if result and result.commit_upto is None:
 468             name += 'commit %2d/%-3d' % (self.commit_upto + 1,
 469                     self.commit_count)
 470
 471         name += target
 472         Print(line + name, newline=False)
 473         length = 16 + len(name)
 474         self.ClearLine(length)
 475
 476     def _GetOutputDir(self, commit_upto):
 477         """Get the name of the output directory for a commit number
 478
 479         The output directory is typically .../<branch>/<commit>.
 480
 481         Args:
 482             commit_upto: Commit number to use (0..self.count-1)
 483         """
 484         commit_dir = None
 485         if self.commits:
 486             commit = self.commits[commit_upto]
 487             subject = commit.subject.translate(trans_valid_chars)
 488             # See _GetOutputSpaceRemovals() which parses this name
 489             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
 490                     self.commit_count, commit.hash, subject[:20]))
 491         elif not self.no_subdirs:
 492             commit_dir = 'current'
 493         if not commit_dir:
 494             return self.base_dir
 495         return os.path.join(self.base_dir, commit_dir)
 496
 497     def GetBuildDir(self, commit_upto, target):
 498         """Get the name of the build directory for a commit number
 499
 500         The build directory is typically .../<branch>/<commit>/<target>.
 501
 502         Args:
 503             commit_upto: Commit number to use (0..self.count-1)
 504             target: Target name
 505         """
 506         output_dir = self._GetOutputDir(commit_upto)
 507         return os.path.join(output_dir, target)
 508
 509     def GetDoneFile(self, commit_upto, target):
 510         """Get the name of the done file for a commit number
 511
 512         Args:
 513             commit_upto: Commit number to use (0..self.count-1)
 514             target: Target name
 515         """
 516         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 517
 518     def GetSizesFile(self, commit_upto, target):
 519         """Get the name of the sizes file for a commit number
 520
 521         Args:
 522             commit_upto: Commit number to use (0..self.count-1)
 523             target: Target name
 524         """
 525         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 526
 527     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 528         """Get the name of the funcsizes file for a commit number and ELF file
 529
 530         Args:
 531             commit_upto: Commit number to use (0..self.count-1)
 532             target: Target name
 533             elf_fname: Filename of elf image
 534         """
 535         return os.path.join(self.GetBuildDir(commit_upto, target),
 536                             '%s.sizes' % elf_fname.replace('/', '-'))
 537
 538     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 539         """Get the name of the objdump file for a commit number and ELF file
 540
 541         Args:
 542             commit_upto: Commit number to use (0..self.count-1)
 543             target: Target name
 544             elf_fname: Filename of elf image
 545         """
 546         return os.path.join(self.GetBuildDir(commit_upto, target),
 547                             '%s.objdump' % elf_fname.replace('/', '-'))
 548
 549     def GetErrFile(self, commit_upto, target):
 550         """Get the name of the err file for a commit number
 551
 552         Args:
 553             commit_upto: Commit number to use (0..self.count-1)
 554             target: Target name
 555         """
 556         output_dir = self.GetBuildDir(commit_upto, target)
 557         return os.path.join(output_dir, 'err')
 558
 559     def FilterErrors(self, lines):
 560         """Filter out errors in which we have no interest
 561
 562         We should probably use map().
 563
 564         Args:
 565             lines: List of error lines, each a string
 566         Returns:
 567             New list with only interesting lines included
 568         """
 569         out_lines = []
 570         for line in lines:
 571             if not self.re_make_err.search(line):
 572                 out_lines.append(line)
 573         return out_lines
 574
 575     def ReadFuncSizes(self, fname, fd):
 576         """Read function sizes from the output of 'nm'
 577
 578         Args:
 579             fd: File containing data to read
 580             fname: Filename we are reading from (just for errors)
 581
 582         Returns:
 583             Dictionary containing size of each function in bytes, indexed by
 584             function name.
 585         """
 586         sym = {}
 587         for line in fd.readlines():
 588             try:
 589                 if line.strip():
 590                     size, type, name = line[:-1].split()
 591             except:
 592                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 593                 continue
 594             if type in 'tTdDbB':
 595                 # function names begin with '.' on 64-bit powerpc
 596                 if '.' in name[1:]:
 597                     name = 'static.' + name.split('.')[0]
 598                 sym[name] = sym.get(name, 0) + int(size, 16)
 599         return sym
 600
 601     def _ProcessConfig(self, fname):
 602         """Read in a .config, autoconf.mk or autoconf.h file
 603
 604         This function handles all config file types. It ignores comments and
 605         any #defines which don't start with CONFIG_.
 606
 607         Args:
 608             fname: Filename to read
 609
 610         Returns:
 611             Dictionary:
 612                 key: Config name (e.g. CONFIG_DM)
 613                 value: Config value (e.g. 1)
 614         """
 615         config = {}
 616         if os.path.exists(fname):
 617             with open(fname) as fd:
 618                 for line in fd:
 619                     line = line.strip()
 620                     if line.startswith('#define'):
 621                         values = line[8:].split(' ', 1)
 622                         if len(values) > 1:
 623                             key, value = values
 624                         else:
 625                             key = values[0]
 626                             value = '1' if self.squash_config_y else ''
 627                         if not key.startswith('CONFIG_'):
 628                             continue
 629                     elif not line or line[0] in ['#', '*', '/']:
 630                         continue
 631                     else:
 632                         key, value = line.split('=', 1)
 633                     if self.squash_config_y and value == 'y':
 634                         value = '1'
 635                     config[key] = value
 636         return config
 637
 638     def _ProcessEnvironment(self, fname):
 639         """Read in a uboot.env file
 640
 641         This function reads in environment variables from a file.
 642
 643         Args:
 644             fname: Filename to read
 645
 646         Returns:
 647             Dictionary:
 648                 key: environment variable (e.g. bootlimit)
 649                 value: value of environment variable (e.g. 1)
 650         """
 651         environment = {}
 652         if os.path.exists(fname):
 653             with open(fname) as fd:
 654                 for line in fd.read().split('\0'):
 655                     try:
 656                         key, value = line.split('=', 1)
 657                         environment[key] = value
 658                     except ValueError:
 659                         # ignore lines we can't parse
 660                         pass
 661         return environment
 662
 663     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 664                         read_config, read_environment):
 665         """Work out the outcome of a build.
 666
 667         Args:
 668             commit_upto: Commit number to check (0..n-1)
 669             target: Target board to check
 670             read_func_sizes: True to read function size information
 671             read_config: True to read .config and autoconf.h files
 672             read_environment: True to read uboot.env files
 673
 674         Returns:
 675             Outcome object
 676         """
 677         done_file = self.GetDoneFile(commit_upto, target)
 678         sizes_file = self.GetSizesFile(commit_upto, target)
 679         sizes = {}
 680         func_sizes = {}
 681         config = {}
 682         environment = {}
 683         if os.path.exists(done_file):
 684             with open(done_file, 'r') as fd:
 685                 try:
 686                     return_code = int(fd.readline())
 687                 except ValueError:
 688                     # The file may be empty due to running out of disk space.
 689                     # Try a rebuild
 690                     return_code = 1
 691                 err_lines = []
 692                 err_file = self.GetErrFile(commit_upto, target)
 693                 if os.path.exists(err_file):
 694                     with open(err_file, 'r') as fd:
 695                         err_lines = self.FilterErrors(fd.readlines())
 696
 697                 # Decide whether the build was ok, failed or created warnings
 698                 if return_code:
 699                     rc = OUTCOME_ERROR
 700                 elif len(err_lines):
 701                     rc = OUTCOME_WARNING
 702                 else:
 703                     rc = OUTCOME_OK
 704
 705                 # Convert size information to our simple format
 706                 if os.path.exists(sizes_file):
 707                     with open(sizes_file, 'r') as fd:
 708                         for line in fd.readlines():
 709                             values = line.split()
 710                             rodata = 0
 711                             if len(values) > 6:
 712                                 rodata = int(values[6], 16)
 713                             size_dict = {
 714                                 'all' : int(values[0]) + int(values[1]) +
 715                                         int(values[2]),
 716                                 'text' : int(values[0]) - rodata,
 717                                 'data' : int(values[1]),
 718                                 'bss' : int(values[2]),
 719                                 'rodata' : rodata,
 720                             }
 721                             sizes[values[5]] = size_dict
 722
 723             if read_func_sizes:
 724                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 725                 for fname in glob.glob(pattern):
 726                     with open(fname, 'r') as fd:
 727                         dict_name = os.path.basename(fname).replace('.sizes',
 728                                                                     '')
 729                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 730
 731             if read_config:
 732                 output_dir = self.GetBuildDir(commit_upto, target)
 733                 for name in self.config_filenames:
 734                     fname = os.path.join(output_dir, name)
 735                     config[name] = self._ProcessConfig(fname)
 736
 737             if read_environment:
 738                 output_dir = self.GetBuildDir(commit_upto, target)
 739                 fname = os.path.join(output_dir, 'uboot.env')
 740                 environment = self._ProcessEnvironment(fname)
 741
 742             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 743                                    environment)
 744
 745         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 746
 747     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 748                          read_config, read_environment):
 749         """Calculate a summary of the results of building a commit.
 750
 751         Args:
 752             board_selected: Dict containing boards to summarise
 753             commit_upto: Commit number to summarize (0..self.count-1)
 754             read_func_sizes: True to read function size information
 755             read_config: True to read .config and autoconf.h files
 756             read_environment: True to read uboot.env files
 757
 758         Returns:
 759             Tuple:
 760                 Dict containing boards which passed building this commit.
 761                     keyed by board.target
 762                 List containing a summary of error lines
 763                 Dict keyed by error line, containing a list of the Board
 764                     objects with that error
 765                 List containing a summary of warning lines
 766                 Dict keyed by error line, containing a list of the Board
 767                     objects with that warning
 768                 Dictionary keyed by board.target. Each value is a dictionary:
 769                     key: filename - e.g. '.config'
 770                     value is itself a dictionary:
 771                         key: config name
 772                         value: config value
 773                 Dictionary keyed by board.target. Each value is a dictionary:
 774                     key: environment variable
 775                     value: value of environment variable
 776         """
 777         def AddLine(lines_summary, lines_boards, line, board):
 778             line = line.rstrip()
 779             if line in lines_boards:
 780                 lines_boards[line].append(board)
 781             else:
 782                 lines_boards[line] = [board]
 783                 lines_summary.append(line)
 784
 785         board_dict = {}
 786         err_lines_summary = []
 787         err_lines_boards = {}
 788         warn_lines_summary = []
 789         warn_lines_boards = {}
 790         config = {}
 791         environment = {}
 792
 793         for board in boards_selected.values():
 794             outcome = self.GetBuildOutcome(commit_upto, board.target,
 795                                            read_func_sizes, read_config,
 796                                            read_environment)
 797             board_dict[board.target] = outcome
 798             last_func = None
 799             last_was_warning = False
 800             for line in outcome.err_lines:
 801                 if line:
 802                     if (self._re_function.match(line) or
 803                             self._re_files.match(line)):
 804                         last_func = line
 805                     else:
 806                         is_warning = (self._re_warning.match(line) or
 807                                       self._re_dtb_warning.match(line))
 808                         is_note = self._re_note.match(line)
 809                         if is_warning or (last_was_warning and is_note):
 810                             if last_func:
 811                                 AddLine(warn_lines_summary, warn_lines_boards,
 812                                         last_func, board)
 813                             AddLine(warn_lines_summary, warn_lines_boards,
 814                                     line, board)
 815                         else:
 816                             if last_func:
 817                                 AddLine(err_lines_summary, err_lines_boards,
 818                                         last_func, board)
 819                             AddLine(err_lines_summary, err_lines_boards,
 820                                     line, board)
 821                         last_was_warning = is_warning
 822                         last_func = None
 823             tconfig = Config(self.config_filenames, board.target)
 824             for fname in self.config_filenames:
 825                 if outcome.config:
 826                     for key, value in outcome.config[fname].items():
 827                         tconfig.Add(fname, key, value)
 828             config[board.target] = tconfig
 829
 830             tenvironment = Environment(board.target)
 831             if outcome.environment:
 832                 for key, value in outcome.environment.items():
 833                     tenvironment.Add(key, value)
 834             environment[board.target] = tenvironment
 835
 836         return (board_dict, err_lines_summary, err_lines_boards,
 837                 warn_lines_summary, warn_lines_boards, config, environment)
 838
 839     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 840         """Add an output to our list of outcomes for each architecture
 841
 842         This simple function adds failing boards (changes) to the
 843         relevant architecture string, so we can print the results out
 844         sorted by architecture.
 845
 846         Args:
 847              board_dict: Dict containing all boards
 848              arch_list: Dict keyed by arch name. Value is a string containing
 849                     a list of board names which failed for that arch.
 850              changes: List of boards to add to arch_list
 851              color: terminal.Colour object
 852         """
 853         done_arch = {}
 854         for target in changes:
 855             if target in board_dict:
 856                 arch = board_dict[target].arch
 857             else:
 858                 arch = 'unknown'
 859             str = self.col.Color(color, ' ' + target)
 860             if not arch in done_arch:
 861                 str = ' %s  %s' % (self.col.Color(color, char), str)
 862                 done_arch[arch] = True
 863             if not arch in arch_list:
 864                 arch_list[arch] = str
 865             else:
 866                 arch_list[arch] += str
 867
 868
 869     def ColourNum(self, num):
 870         color = self.col.RED if num > 0 else self.col.GREEN
 871         if num == 0:
 872             return '0'
 873         return self.col.Color(color, str(num))
 874
 875     def ResetResultSummary(self, board_selected):
 876         """Reset the results summary ready for use.
 877
 878         Set up the base board list to be all those selected, and set the
 879         error lines to empty.
 880
 881         Following this, calls to PrintResultSummary() will use this
 882         information to work out what has changed.
 883
 884         Args:
 885             board_selected: Dict containing boards to summarise, keyed by
 886                 board.target
 887         """
 888         self._base_board_dict = {}
 889         for board in board_selected:
 890             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 891                                                            {})
 892         self._base_err_lines = []
 893         self._base_warn_lines = []
 894         self._base_err_line_boards = {}
 895         self._base_warn_line_boards = {}
 896         self._base_config = None
 897         self._base_environment = None
 898
 899     def PrintFuncSizeDetail(self, fname, old, new):
 900         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 901         delta, common = [], {}
 902
 903         for a in old:
 904             if a in new:
 905                 common[a] = 1
 906
 907         for name in old:
 908             if name not in common:
 909                 remove += 1
 910                 down += old[name]
 911                 delta.append([-old[name], name])
 912
 913         for name in new:
 914             if name not in common:
 915                 add += 1
 916                 up += new[name]
 917                 delta.append([new[name], name])
 918
 919         for name in common:
 920                 diff = new.get(name, 0) - old.get(name, 0)
 921                 if diff > 0:
 922                     grow, up = grow + 1, up + diff
 923                 elif diff < 0:
 924                     shrink, down = shrink + 1, down - diff
 925                 delta.append([diff, name])
 926
 927         delta.sort()
 928         delta.reverse()
 929
 930         args = [add, -remove, grow, -shrink, up, -down, up - down]
 931         if max(args) == 0 and min(args) == 0:
 932             return
 933         args = [self.ColourNum(x) for x in args]
 934         indent = ' ' * 15
 935         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
 936               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
 937         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
 938                                          'delta'))
 939         for diff, name in delta:
 940             if diff:
 941                 color = self.col.RED if diff > 0 else self.col.GREEN
 942                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
 943                         old.get(name, '-'), new.get(name,'-'), diff)
 944                 Print(msg, colour=color)
 945
 946
 947     def PrintSizeDetail(self, target_list, show_bloat):
 948         """Show details size information for each board
 949
 950         Args:
 951             target_list: List of targets, each a dict containing:
 952                     'target': Target name
 953                     'total_diff': Total difference in bytes across all areas
 954                     <part_name>: Difference for that part
 955             show_bloat: Show detail for each function
 956         """
 957         targets_by_diff = sorted(target_list, reverse=True,
 958         key=lambda x: x['_total_diff'])
 959         for result in targets_by_diff:
 960             printed_target = False
 961             for name in sorted(result):
 962                 diff = result[name]
 963                 if name.startswith('_'):
 964                     continue
 965                 if diff != 0:
 966                     color = self.col.RED if diff > 0 else self.col.GREEN
 967                 msg = ' %s %+d' % (name, diff)
 968                 if not printed_target:
 969                     Print('%10s  %-15s:' % ('', result['_target']),
 970                           newline=False)
 971                     printed_target = True
 972                 Print(msg, colour=color, newline=False)
 973             if printed_target:
 974                 Print()
 975                 if show_bloat:
 976                     target = result['_target']
 977                     outcome = result['_outcome']
 978                     base_outcome = self._base_board_dict[target]
 979                     for fname in outcome.func_sizes:
 980                         self.PrintFuncSizeDetail(fname,
 981                                                  base_outcome.func_sizes[fname],
 982                                                  outcome.func_sizes[fname])
 983
 984
 985     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
 986                          show_bloat):
 987         """Print a summary of image sizes broken down by section.
 988
 989         The summary takes the form of one line per architecture. The
 990         line contains deltas for each of the sections (+ means the section
 991         got bigger, - means smaller). The numbers are the average number
 992         of bytes that a board in this section increased by.
 993
 994         For example:
 995            powerpc: (622 boards)   text -0.0
 996           arm: (285 boards)   text -0.0
 997           nds32: (3 boards)   text -8.0
 998
 999         Args:
1000             board_selected: Dict containing boards to summarise, keyed by
1001                 board.target
1002             board_dict: Dict containing boards for which we built this
1003                 commit, keyed by board.target. The value is an Outcome object.
1004             show_detail: Show size delta detail for each board
1005             show_bloat: Show detail for each function
1006         """
1007         arch_list = {}
1008         arch_count = {}
1009
1010         # Calculate changes in size for different image parts
1011         # The previous sizes are in Board.sizes, for each board
1012         for target in board_dict:
1013             if target not in board_selected:
1014                 continue
1015             base_sizes = self._base_board_dict[target].sizes
1016             outcome = board_dict[target]
1017             sizes = outcome.sizes
1018
1019             # Loop through the list of images, creating a dict of size
1020             # changes for each image/part. We end up with something like
1021             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1022             # which means that U-Boot data increased by 5 bytes and SPL
1023             # text decreased by 4.
1024             err = {'_target' : target}
1025             for image in sizes:
1026                 if image in base_sizes:
1027                     base_image = base_sizes[image]
1028                     # Loop through the text, data, bss parts
1029                     for part in sorted(sizes[image]):
1030                         diff = sizes[image][part] - base_image[part]
1031                         col = None
1032                         if diff:
1033                             if image == 'u-boot':
1034                                 name = part
1035                             else:
1036                                 name = image + ':' + part
1037                             err[name] = diff
1038             arch = board_selected[target].arch
1039             if not arch in arch_count:
1040                 arch_count[arch] = 1
1041             else:
1042                 arch_count[arch] += 1
1043             if not sizes:
1044                 pass    # Only add to our list when we have some stats
1045             elif not arch in arch_list:
1046                 arch_list[arch] = [err]
1047             else:
1048                 arch_list[arch].append(err)
1049
1050         # We now have a list of image size changes sorted by arch
1051         # Print out a summary of these
1052         for arch, target_list in arch_list.items():
1053             # Get total difference for each type
1054             totals = {}
1055             for result in target_list:
1056                 total = 0
1057                 for name, diff in result.items():
1058                     if name.startswith('_'):
1059                         continue
1060                     total += diff
1061                     if name in totals:
1062                         totals[name] += diff
1063                     else:
1064                         totals[name] = diff
1065                 result['_total_diff'] = total
1066                 result['_outcome'] = board_dict[result['_target']]
1067
1068             count = len(target_list)
1069             printed_arch = False
1070             for name in sorted(totals):
1071                 diff = totals[name]
1072                 if diff:
1073                     # Display the average difference in this name for this
1074                     # architecture
1075                     avg_diff = float(diff) / count
1076                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1077                     msg = ' %s %+1.1f' % (name, avg_diff)
1078                     if not printed_arch:
1079                         Print('%10s: (for %d/%d boards)' % (arch, count,
1080                               arch_count[arch]), newline=False)
1081                         printed_arch = True
1082                     Print(msg, colour=color, newline=False)
1083
1084             if printed_arch:
1085                 Print()
1086                 if show_detail:
1087                     self.PrintSizeDetail(target_list, show_bloat)
1088
1089
1090     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1091                            err_line_boards, warn_lines, warn_line_boards,
1092                            config, environment, show_sizes, show_detail,
1093                            show_bloat, show_config, show_environment):
1094         """Compare results with the base results and display delta.
1095
1096         Only boards mentioned in board_selected will be considered. This
1097         function is intended to be called repeatedly with the results of
1098         each commit. It therefore shows a 'diff' between what it saw in
1099         the last call and what it sees now.
1100
1101         Args:
1102             board_selected: Dict containing boards to summarise, keyed by
1103                 board.target
1104             board_dict: Dict containing boards for which we built this
1105                 commit, keyed by board.target. The value is an Outcome object.
1106             err_lines: A list of errors for this commit, or [] if there is
1107                 none, or we don't want to print errors
1108             err_line_boards: Dict keyed by error line, containing a list of
1109                 the Board objects with that error
1110             warn_lines: A list of warnings for this commit, or [] if there is
1111                 none, or we don't want to print errors
1112             warn_line_boards: Dict keyed by warning line, containing a list of
1113                 the Board objects with that warning
1114             config: Dictionary keyed by filename - e.g. '.config'. Each
1115                     value is itself a dictionary:
1116                         key: config name
1117                         value: config value
1118             environment: Dictionary keyed by environment variable, Each
1119                      value is the value of environment variable.
1120             show_sizes: Show image size deltas
1121             show_detail: Show size delta detail for each board if show_sizes
1122             show_bloat: Show detail for each function
1123             show_config: Show config changes
1124             show_environment: Show environment changes
1125         """
1126         def _BoardList(line, line_boards):
1127             """Helper function to get a line of boards containing a line
1128
1129             Args:
1130                 line: Error line to search for
1131             Return:
1132                 String containing a list of boards with that error line, or
1133                 '' if the user has not requested such a list
1134             """
1135             if self._list_error_boards:
1136                 names = []
1137                 for board in line_boards[line]:
1138                     if not board.target in names:
1139                         names.append(board.target)
1140                 names_str = '(%s) ' % ','.join(names)
1141             else:
1142                 names_str = ''
1143             return names_str
1144
1145         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1146                             char):
1147             better_lines = []
1148             worse_lines = []
1149             for line in lines:
1150                 if line not in base_lines:
1151                     worse_lines.append(char + '+' +
1152                             _BoardList(line, line_boards) + line)
1153             for line in base_lines:
1154                 if line not in lines:
1155                     better_lines.append(char + '-' +
1156                             _BoardList(line, base_line_boards) + line)
1157             return better_lines, worse_lines
1158
1159         def _CalcConfig(delta, name, config):
1160             """Calculate configuration changes
1161
1162             Args:
1163                 delta: Type of the delta, e.g. '+'
1164                 name: name of the file which changed (e.g. .config)
1165                 config: configuration change dictionary
1166                     key: config name
1167                     value: config value
1168             Returns:
1169                 String containing the configuration changes which can be
1170                     printed
1171             """
1172             out = ''
1173             for key in sorted(config.keys()):
1174                 out += '%s=%s ' % (key, config[key])
1175             return '%s %s: %s' % (delta, name, out)
1176
1177         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1178             """Add changes in configuration to a list
1179
1180             Args:
1181                 lines: list to add to
1182                 name: config file name
1183                 config_plus: configurations added, dictionary
1184                     key: config name
1185                     value: config value
1186                 config_minus: configurations removed, dictionary
1187                     key: config name
1188                     value: config value
1189                 config_change: configurations changed, dictionary
1190                     key: config name
1191                     value: config value
1192             """
1193             if config_plus:
1194                 lines.append(_CalcConfig('+', name, config_plus))
1195             if config_minus:
1196                 lines.append(_CalcConfig('-', name, config_minus))
1197             if config_change:
1198                 lines.append(_CalcConfig('c', name, config_change))
1199
1200         def _OutputConfigInfo(lines):
1201             for line in lines:
1202                 if not line:
1203                     continue
1204                 if line[0] == '+':
1205                     col = self.col.GREEN
1206                 elif line[0] == '-':
1207                     col = self.col.RED
1208                 elif line[0] == 'c':
1209                     col = self.col.YELLOW
1210                 Print('   ' + line, newline=True, colour=col)
1211
1212
1213         ok_boards = []      # List of boards fixed since last commit
1214         warn_boards = []    # List of boards with warnings since last commit
1215         err_boards = []     # List of new broken boards since last commit
1216         new_boards = []     # List of boards that didn't exist last time
1217         unknown_boards = [] # List of boards that were not built
1218
1219         for target in board_dict:
1220             if target not in board_selected:
1221                 continue
1222
1223             # If the board was built last time, add its outcome to a list
1224             if target in self._base_board_dict:
1225                 base_outcome = self._base_board_dict[target].rc
1226                 outcome = board_dict[target]
1227                 if outcome.rc == OUTCOME_UNKNOWN:
1228                     unknown_boards.append(target)
1229                 elif outcome.rc < base_outcome:
1230                     if outcome.rc == OUTCOME_WARNING:
1231                         warn_boards.append(target)
1232                     else:
1233                         ok_boards.append(target)
1234                 elif outcome.rc > base_outcome:
1235                     if outcome.rc == OUTCOME_WARNING:
1236                         warn_boards.append(target)
1237                     else:
1238                         err_boards.append(target)
1239             else:
1240                 new_boards.append(target)
1241
1242         # Get a list of errors that have appeared, and disappeared
1243         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1244                 self._base_err_line_boards, err_lines, err_line_boards, '')
1245         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1246                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1247
1248         # Display results by arch
1249         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1250                 worse_err, better_err, worse_warn, better_warn)):
1251             arch_list = {}
1252             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1253                     self.col.GREEN)
1254             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1255                     self.col.YELLOW)
1256             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1257                     self.col.RED)
1258             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1259             if self._show_unknown:
1260                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1261                         self.col.MAGENTA)
1262             for arch, target_list in arch_list.items():
1263                 Print('%10s: %s' % (arch, target_list))
1264                 self._error_lines += 1
1265             if better_err:
1266                 Print('\n'.join(better_err), colour=self.col.GREEN)
1267                 self._error_lines += 1
1268             if worse_err:
1269                 Print('\n'.join(worse_err), colour=self.col.RED)
1270                 self._error_lines += 1
1271             if better_warn:
1272                 Print('\n'.join(better_warn), colour=self.col.CYAN)
1273                 self._error_lines += 1
1274             if worse_warn:
1275                 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
1276                 self._error_lines += 1
1277
1278         if show_sizes:
1279             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1280                                   show_bloat)
1281
1282         if show_environment and self._base_environment:
1283             lines = []
1284
1285             for target in board_dict:
1286                 if target not in board_selected:
1287                     continue
1288
1289                 tbase = self._base_environment[target]
1290                 tenvironment = environment[target]
1291                 environment_plus = {}
1292                 environment_minus = {}
1293                 environment_change = {}
1294                 base = tbase.environment
1295                 for key, value in tenvironment.environment.items():
1296                     if key not in base:
1297                         environment_plus[key] = value
1298                 for key, value in base.items():
1299                     if key not in tenvironment.environment:
1300                         environment_minus[key] = value
1301                 for key, value in base.items():
1302                     new_value = tenvironment.environment.get(key)
1303                     if new_value and value != new_value:
1304                         desc = '%s -> %s' % (value, new_value)
1305                         environment_change[key] = desc
1306
1307                 _AddConfig(lines, target, environment_plus, environment_minus,
1308                            environment_change)
1309
1310             _OutputConfigInfo(lines)
1311
1312         if show_config and self._base_config:
1313             summary = {}
1314             arch_config_plus = {}
1315             arch_config_minus = {}
1316             arch_config_change = {}
1317             arch_list = []
1318
1319             for target in board_dict:
1320                 if target not in board_selected:
1321                     continue
1322                 arch = board_selected[target].arch
1323                 if arch not in arch_list:
1324                     arch_list.append(arch)
1325
1326             for arch in arch_list:
1327                 arch_config_plus[arch] = {}
1328                 arch_config_minus[arch] = {}
1329                 arch_config_change[arch] = {}
1330                 for name in self.config_filenames:
1331                     arch_config_plus[arch][name] = {}
1332                     arch_config_minus[arch][name] = {}
1333                     arch_config_change[arch][name] = {}
1334
1335             for target in board_dict:
1336                 if target not in board_selected:
1337                     continue
1338
1339                 arch = board_selected[target].arch
1340
1341                 all_config_plus = {}
1342                 all_config_minus = {}
1343                 all_config_change = {}
1344                 tbase = self._base_config[target]
1345                 tconfig = config[target]
1346                 lines = []
1347                 for name in self.config_filenames:
1348                     if not tconfig.config[name]:
1349                         continue
1350                     config_plus = {}
1351                     config_minus = {}
1352                     config_change = {}
1353                     base = tbase.config[name]
1354                     for key, value in tconfig.config[name].items():
1355                         if key not in base:
1356                             config_plus[key] = value
1357                             all_config_plus[key] = value
1358                     for key, value in base.items():
1359                         if key not in tconfig.config[name]:
1360                             config_minus[key] = value
1361                             all_config_minus[key] = value
1362                     for key, value in base.items():
1363                         new_value = tconfig.config.get(key)
1364                         if new_value and value != new_value:
1365                             desc = '%s -> %s' % (value, new_value)
1366                             config_change[key] = desc
1367                             all_config_change[key] = desc
1368
1369                     arch_config_plus[arch][name].update(config_plus)
1370                     arch_config_minus[arch][name].update(config_minus)
1371                     arch_config_change[arch][name].update(config_change)
1372
1373                     _AddConfig(lines, name, config_plus, config_minus,
1374                                config_change)
1375                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1376                            all_config_change)
1377                 summary[target] = '\n'.join(lines)
1378
1379             lines_by_target = {}
1380             for target, lines in summary.items():
1381                 if lines in lines_by_target:
1382                     lines_by_target[lines].append(target)
1383                 else:
1384                     lines_by_target[lines] = [target]
1385
1386             for arch in arch_list:
1387                 lines = []
1388                 all_plus = {}
1389                 all_minus = {}
1390                 all_change = {}
1391                 for name in self.config_filenames:
1392                     all_plus.update(arch_config_plus[arch][name])
1393                     all_minus.update(arch_config_minus[arch][name])
1394                     all_change.update(arch_config_change[arch][name])
1395                     _AddConfig(lines, name, arch_config_plus[arch][name],
1396                                arch_config_minus[arch][name],
1397                                arch_config_change[arch][name])
1398                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1399                 #arch_summary[target] = '\n'.join(lines)
1400                 if lines:
1401                     Print('%s:' % arch)
1402                     _OutputConfigInfo(lines)
1403
1404             for lines, targets in lines_by_target.items():
1405                 if not lines:
1406                     continue
1407                 Print('%s :' % ' '.join(sorted(targets)))
1408                 _OutputConfigInfo(lines.split('\n'))
1409
1410
1411         # Save our updated information for the next call to this function
1412         self._base_board_dict = board_dict
1413         self._base_err_lines = err_lines
1414         self._base_warn_lines = warn_lines
1415         self._base_err_line_boards = err_line_boards
1416         self._base_warn_line_boards = warn_line_boards
1417         self._base_config = config
1418         self._base_environment = environment
1419
1420         # Get a list of boards that did not get built, if needed
1421         not_built = []
1422         for board in board_selected:
1423             if not board in board_dict:
1424                 not_built.append(board)
1425         if not_built:
1426             Print("Boards not built (%d): %s" % (len(not_built),
1427                   ', '.join(not_built)))
1428
1429     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1430             (board_dict, err_lines, err_line_boards, warn_lines,
1431              warn_line_boards, config, environment) = self.GetResultSummary(
1432                     board_selected, commit_upto,
1433                     read_func_sizes=self._show_bloat,
1434                     read_config=self._show_config,
1435                     read_environment=self._show_environment)
1436             if commits:
1437                 msg = '%02d: %s' % (commit_upto + 1,
1438                         commits[commit_upto].subject)
1439                 Print(msg, colour=self.col.BLUE)
1440             self.PrintResultSummary(board_selected, board_dict,
1441                     err_lines if self._show_errors else [], err_line_boards,
1442                     warn_lines if self._show_errors else [], warn_line_boards,
1443                     config, environment, self._show_sizes, self._show_detail,
1444                     self._show_bloat, self._show_config, self._show_environment)
1445
1446     def ShowSummary(self, commits, board_selected):
1447         """Show a build summary for U-Boot for a given board list.
1448
1449         Reset the result summary, then repeatedly call GetResultSummary on
1450         each commit's results, then display the differences we see.
1451
1452         Args:
1453             commit: Commit objects to summarise
1454             board_selected: Dict containing boards to summarise
1455         """
1456         self.commit_count = len(commits) if commits else 1
1457         self.commits = commits
1458         self.ResetResultSummary(board_selected)
1459         self._error_lines = 0
1460
1461         for commit_upto in range(0, self.commit_count, self._step):
1462             self.ProduceResultSummary(commit_upto, commits, board_selected)
1463         if not self._error_lines:
1464             Print('(no errors to report)', colour=self.col.GREEN)
1465
1466
1467     def SetupBuild(self, board_selected, commits):
1468         """Set up ready to start a build.
1469
1470         Args:
1471             board_selected: Selected boards to build
1472             commits: Selected commits to build
1473         """
1474         # First work out how many commits we will build
1475         count = (self.commit_count + self._step - 1) // self._step
1476         self.count = len(board_selected) * count
1477         self.upto = self.warned = self.fail = 0
1478         self._timestamps = collections.deque()
1479
1480     def GetThreadDir(self, thread_num):
1481         """Get the directory path to the working dir for a thread.
1482
1483         Args:
1484             thread_num: Number of thread to check.
1485         """
1486         if self.work_in_output:
1487             return self._working_dir
1488         return os.path.join(self._working_dir, '%02d' % thread_num)
1489
1490     def _PrepareThread(self, thread_num, setup_git):
1491         """Prepare the working directory for a thread.
1492
1493         This clones or fetches the repo into the thread's work directory.
1494
1495         Args:
1496             thread_num: Thread number (0, 1, ...)
1497             setup_git: True to set up a git repo clone
1498         """
1499         thread_dir = self.GetThreadDir(thread_num)
1500         builderthread.Mkdir(thread_dir)
1501         git_dir = os.path.join(thread_dir, '.git')
1502
1503         # Clone the repo if it doesn't already exist
1504         # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1505         # we have a private index but uses the origin repo's contents?
1506         if setup_git and self.git_dir:
1507             src_dir = os.path.abspath(self.git_dir)
1508             if os.path.exists(git_dir):
1509                 gitutil.Fetch(git_dir, thread_dir)
1510             else:
1511                 Print('\rCloning repo for thread %d' % thread_num,
1512                       newline=False)
1513                 gitutil.Clone(src_dir, thread_dir)
1514                 Print('\r%s\r' % (' ' * 30), newline=False)
1515
1516     def _PrepareWorkingSpace(self, max_threads, setup_git):
1517         """Prepare the working directory for use.
1518
1519         Set up the git repo for each thread.
1520
1521         Args:
1522             max_threads: Maximum number of threads we expect to need.
1523             setup_git: True to set up a git repo clone
1524         """
1525         builderthread.Mkdir(self._working_dir)
1526         for thread in range(max_threads):
1527             self._PrepareThread(thread, setup_git)
1528
1529     def _GetOutputSpaceRemovals(self):
1530         """Get the output directories ready to receive files.
1531
1532         Figure out what needs to be deleted in the output directory before it
1533         can be used. We only delete old buildman directories which have the
1534         expected name pattern. See _GetOutputDir().
1535
1536         Returns:
1537             List of full paths of directories to remove
1538         """
1539         if not self.commits:
1540             return
1541         dir_list = []
1542         for commit_upto in range(self.commit_count):
1543             dir_list.append(self._GetOutputDir(commit_upto))
1544
1545         to_remove = []
1546         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1547             if dirname not in dir_list:
1548                 leaf = dirname[len(self.base_dir) + 1:]
1549                 m =  re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1550                 if m:
1551                     to_remove.append(dirname)
1552         return to_remove
1553
1554     def _PrepareOutputSpace(self):
1555         """Get the output directories ready to receive files.
1556
1557         We delete any output directories which look like ones we need to
1558         create. Having left over directories is confusing when the user wants
1559         to check the output manually.
1560         """
1561         to_remove = self._GetOutputSpaceRemovals()
1562         if to_remove:
1563             Print('Removing %d old build directories...' % len(to_remove),
1564                   newline=False)
1565             for dirname in to_remove:
1566                 shutil.rmtree(dirname)
1567             Print('done')
1568
1569     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1570         """Build all commits for a list of boards
1571
1572         Args:
1573             commits: List of commits to be build, each a Commit object
1574             boards_selected: Dict of selected boards, key is target name,
1575                     value is Board object
1576             keep_outputs: True to save build output files
1577             verbose: Display build results as they are completed
1578         Returns:
1579             Tuple containing:
1580                 - number of boards that failed to build
1581                 - number of boards that issued warnings
1582         """
1583         self.commit_count = len(commits) if commits else 1
1584         self.commits = commits
1585         self._verbose = verbose
1586
1587         self.ResetResultSummary(board_selected)
1588         builderthread.Mkdir(self.base_dir, parents = True)
1589         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1590                 commits is not None)
1591         self._PrepareOutputSpace()
1592         Print('\rStarting build...', newline=False)
1593         self.SetupBuild(board_selected, commits)
1594         self.ProcessResult(None)
1595
1596         # Create jobs to build all commits for each board
1597         for brd in board_selected.values():
1598             job = builderthread.BuilderJob()
1599             job.board = brd
1600             job.commits = commits
1601             job.keep_outputs = keep_outputs
1602             job.work_in_output = self.work_in_output
1603             job.step = self._step
1604             self.queue.put(job)
1605
1606         term = threading.Thread(target=self.queue.join)
1607         term.setDaemon(True)
1608         term.start()
1609         while term.isAlive():
1610             term.join(100)
1611
1612         # Wait until we have processed all output
1613         self.out_queue.join()
1614         Print()
1615         self.ClearLine(0)
1616         return (self.fail, self.warned)