tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 from buildman import builderthread
  21 from buildman import toolchain
  22 from patman import command
  23 from patman import gitutil
  24 from patman import terminal
  25 from patman.terminal import Print
  26
  27 """
  28 Theory of Operation
  29
  30 Please see README for user documentation, and you should be familiar with
  31 that before trying to make sense of this.
  32
  33 Buildman works by keeping the machine as busy as possible, building different
  34 commits for different boards on multiple CPUs at once.
  35
  36 The source repo (self.git_dir) contains all the commits to be built. Each
  37 thread works on a single board at a time. It checks out the first commit,
  38 configures it for that board, then builds it. Then it checks out the next
  39 commit and builds it (typically without re-configuring). When it runs out
  40 of commits, it gets another job from the builder and starts again with that
  41 board.
  42
  43 Clearly the builder threads could work either way - they could check out a
  44 commit and then built it for all boards. Using separate directories for each
  45 commit/board pair they could leave their build product around afterwards
  46 also.
  47
  48 The intent behind building a single board for multiple commits, is to make
  49 use of incremental builds. Since each commit is built incrementally from
  50 the previous one, builds are faster. Reconfiguring for a different board
  51 removes all intermediate object files.
  52
  53 Many threads can be working at once, but each has its own working directory.
  54 When a thread finishes a build, it puts the output files into a result
  55 directory.
  56
  57 The base directory used by buildman is normally '../<branch>', i.e.
  58 a directory higher than the source repository and named after the branch
  59 being built.
  60
  61 Within the base directory, we have one subdirectory for each commit. Within
  62 that is one subdirectory for each board. Within that is the build output for
  63 that commit/board combination.
  64
  65 Buildman also create working directories for each thread, in a .bm-work/
  66 subdirectory in the base dir.
  67
  68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  69 'seaboard', and say that us-net has two commits. We will have directories
  70 like this:
  71
  72 us-net/             base directory
  73     01_of_02_g4ed4ebc_net--Add-tftp-speed-/
  74         sandbox/
  75             u-boot.bin
  76         seaboard/
  77             u-boot.bin
  78     02_of_02_g4ed4ebc_net--Check-tftp-comp/
  79         sandbox/
  80             u-boot.bin
  81         seaboard/
  82             u-boot.bin
  83     .bm-work/
  84         00/         working directory for thread 0 (contains source checkout)
  85             build/  build output
  86         01/         working directory for thread 1
  87             build/  build output
  88         ...
  89 u-boot/             source directory
  90     .git/           repository
  91 """
  92
  93 """Holds information about a particular error line we are outputing
  94
  95    char: Character representation: '+': error, '-': fixed error, 'w+': warning,
  96        'w-' = fixed warning
  97    boards: List of Board objects which have line in the error/warning output
  98    errline: The text of the error line
  99 """
 100 ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
 101
 102 # Possible build outcomes
 103 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 104
 105 # Translate a commit subject into a valid filename (and handle unicode)
 106 trans_valid_chars = str.maketrans('/: ', '---')
 107
 108 BASE_CONFIG_FILENAMES = [
 109     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 110 ]
 111
 112 EXTRA_CONFIG_FILENAMES = [
 113     '.config', '.config-spl', '.config-tpl',
 114     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 115     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 116 ]
 117
 118 class Config:
 119     """Holds information about configuration settings for a board."""
 120     def __init__(self, config_filename, target):
 121         self.target = target
 122         self.config = {}
 123         for fname in config_filename:
 124             self.config[fname] = {}
 125
 126     def Add(self, fname, key, value):
 127         self.config[fname][key] = value
 128
 129     def __hash__(self):
 130         val = 0
 131         for fname in self.config:
 132             for key, value in self.config[fname].items():
 133                 print(key, value)
 134                 val = val ^ hash(key) & hash(value)
 135         return val
 136
 137 class Environment:
 138     """Holds information about environment variables for a board."""
 139     def __init__(self, target):
 140         self.target = target
 141         self.environment = {}
 142
 143     def Add(self, key, value):
 144         self.environment[key] = value
 145
 146 class Builder:
 147     """Class for building U-Boot for a particular commit.
 148
 149     Public members: (many should ->private)
 150         already_done: Number of builds already completed
 151         base_dir: Base directory to use for builder
 152         checkout: True to check out source, False to skip that step.
 153             This is used for testing.
 154         col: terminal.Color() object
 155         count: Number of commits to build
 156         do_make: Method to call to invoke Make
 157         fail: Number of builds that failed due to error
 158         force_build: Force building even if a build already exists
 159         force_config_on_failure: If a commit fails for a board, disable
 160             incremental building for the next commit we build for that
 161             board, so that we will see all warnings/errors again.
 162         force_build_failures: If a previously-built build (i.e. built on
 163             a previous run of buildman) is marked as failed, rebuild it.
 164         git_dir: Git directory containing source repository
 165         num_jobs: Number of jobs to run at once (passed to make as -j)
 166         num_threads: Number of builder threads to run
 167         out_queue: Queue of results to process
 168         re_make_err: Compiled regular expression for ignore_lines
 169         queue: Queue of jobs to run
 170         threads: List of active threads
 171         toolchains: Toolchains object to use for building
 172         upto: Current commit number we are building (0.count-1)
 173         warned: Number of builds that produced at least one warning
 174         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 175             incremental building, where buildman reconfigures on the first
 176             commit for a baord, and then just does an incremental build for
 177             the following commits. In fact buildman will reconfigure and
 178             retry for any failing commits, so generally the only effect of
 179             this option is to slow things down.
 180         in_tree: Build U-Boot in-tree instead of specifying an output
 181             directory separate from the source code. This option is really
 182             only useful for testing in-tree builds.
 183         work_in_output: Use the output directory as the work directory and
 184             don't write to a separate output directory.
 185
 186     Private members:
 187         _base_board_dict: Last-summarised Dict of boards
 188         _base_err_lines: Last-summarised list of errors
 189         _base_warn_lines: Last-summarised list of warnings
 190         _build_period_us: Time taken for a single build (float object).
 191         _complete_delay: Expected delay until completion (timedelta)
 192         _next_delay_update: Next time we plan to display a progress update
 193                 (datatime)
 194         _show_unknown: Show unknown boards (those not built) in summary
 195         _start_time: Start time for the build
 196         _timestamps: List of timestamps for the completion of the last
 197             last _timestamp_count builds. Each is a datetime object.
 198         _timestamp_count: Number of timestamps to keep in our list.
 199         _working_dir: Base working directory containing all threads
 200     """
 201     class Outcome:
 202         """Records a build outcome for a single make invocation
 203
 204         Public Members:
 205             rc: Outcome value (OUTCOME_...)
 206             err_lines: List of error lines or [] if none
 207             sizes: Dictionary of image size information, keyed by filename
 208                 - Each value is itself a dictionary containing
 209                     values for 'text', 'data' and 'bss', being the integer
 210                     size in bytes of each section.
 211             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 212                     value is itself a dictionary:
 213                         key: function name
 214                         value: Size of function in bytes
 215             config: Dictionary keyed by filename - e.g. '.config'. Each
 216                     value is itself a dictionary:
 217                         key: config name
 218                         value: config value
 219             environment: Dictionary keyed by environment variable, Each
 220                      value is the value of environment variable.
 221         """
 222         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 223                      environment):
 224             self.rc = rc
 225             self.err_lines = err_lines
 226             self.sizes = sizes
 227             self.func_sizes = func_sizes
 228             self.config = config
 229             self.environment = environment
 230
 231     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 232                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 233                  no_subdirs=False, full_path=False, verbose_build=False,
 234                  mrproper=False, per_board_out_dir=False,
 235                  config_only=False, squash_config_y=False,
 236                  warnings_as_errors=False, work_in_output=False):
 237         """Create a new Builder object
 238
 239         Args:
 240             toolchains: Toolchains object to use for building
 241             base_dir: Base directory to use for builder
 242             git_dir: Git directory containing source repository
 243             num_threads: Number of builder threads to run
 244             num_jobs: Number of jobs to run at once (passed to make as -j)
 245             gnu_make: the command name of GNU Make.
 246             checkout: True to check out source, False to skip that step.
 247                 This is used for testing.
 248             show_unknown: Show unknown boards (those not built) in summary
 249             step: 1 to process every commit, n to process every nth commit
 250             no_subdirs: Don't create subdirectories when building current
 251                 source for a single board
 252             full_path: Return the full path in CROSS_COMPILE and don't set
 253                 PATH
 254             verbose_build: Run build with V=1 and don't use 'make -s'
 255             mrproper: Always run 'make mrproper' when configuring
 256             per_board_out_dir: Build in a separate persistent directory per
 257                 board rather than a thread-specific directory
 258             config_only: Only configure each build, don't build it
 259             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 260             warnings_as_errors: Treat all compiler warnings as errors
 261             work_in_output: Use the output directory as the work directory and
 262                 don't write to a separate output directory.
 263         """
 264         self.toolchains = toolchains
 265         self.base_dir = base_dir
 266         if work_in_output:
 267             self._working_dir = base_dir
 268         else:
 269             self._working_dir = os.path.join(base_dir, '.bm-work')
 270         self.threads = []
 271         self.do_make = self.Make
 272         self.gnu_make = gnu_make
 273         self.checkout = checkout
 274         self.num_threads = num_threads
 275         self.num_jobs = num_jobs
 276         self.already_done = 0
 277         self.force_build = False
 278         self.git_dir = git_dir
 279         self._show_unknown = show_unknown
 280         self._timestamp_count = 10
 281         self._build_period_us = None
 282         self._complete_delay = None
 283         self._next_delay_update = datetime.now()
 284         self._start_time = datetime.now()
 285         self.force_config_on_failure = True
 286         self.force_build_failures = False
 287         self.force_reconfig = False
 288         self._step = step
 289         self.in_tree = False
 290         self._error_lines = 0
 291         self.no_subdirs = no_subdirs
 292         self.full_path = full_path
 293         self.verbose_build = verbose_build
 294         self.config_only = config_only
 295         self.squash_config_y = squash_config_y
 296         self.config_filenames = BASE_CONFIG_FILENAMES
 297         self.work_in_output = work_in_output
 298         if not self.squash_config_y:
 299             self.config_filenames += EXTRA_CONFIG_FILENAMES
 300
 301         self.warnings_as_errors = warnings_as_errors
 302         self.col = terminal.Color()
 303
 304         self._re_function = re.compile('(.*): In function.*')
 305         self._re_files = re.compile('In file included from.*')
 306         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 307         self._re_dtb_warning = re.compile('(.*): Warning .*')
 308         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 309         self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
 310                                                 re.MULTILINE | re.DOTALL)
 311
 312         self.queue = queue.Queue()
 313         self.out_queue = queue.Queue()
 314         for i in range(self.num_threads):
 315             t = builderthread.BuilderThread(self, i, mrproper,
 316                     per_board_out_dir)
 317             t.setDaemon(True)
 318             t.start()
 319             self.threads.append(t)
 320
 321         t = builderthread.ResultThread(self)
 322         t.setDaemon(True)
 323         t.start()
 324         self.threads.append(t)
 325
 326         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 327         self.re_make_err = re.compile('|'.join(ignore_lines))
 328
 329         # Handle existing graceful with SIGINT / Ctrl-C
 330         signal.signal(signal.SIGINT, self.signal_handler)
 331
 332     def __del__(self):
 333         """Get rid of all threads created by the builder"""
 334         for t in self.threads:
 335             del t
 336
 337     def signal_handler(self, signal, frame):
 338         sys.exit(1)
 339
 340     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 341                           show_detail=False, show_bloat=False,
 342                           list_error_boards=False, show_config=False,
 343                           show_environment=False, filter_dtb_warnings=False,
 344                           filter_migration_warnings=False):
 345         """Setup display options for the builder.
 346
 347         Args:
 348             show_errors: True to show summarised error/warning info
 349             show_sizes: Show size deltas
 350             show_detail: Show size delta detail for each board if show_sizes
 351             show_bloat: Show detail for each function
 352             list_error_boards: Show the boards which caused each error/warning
 353             show_config: Show config deltas
 354             show_environment: Show environment deltas
 355             filter_dtb_warnings: Filter out any warnings from the device-tree
 356                 compiler
 357             filter_migration_warnings: Filter out any warnings about migrating
 358                 a board to driver model
 359         """
 360         self._show_errors = show_errors
 361         self._show_sizes = show_sizes
 362         self._show_detail = show_detail
 363         self._show_bloat = show_bloat
 364         self._list_error_boards = list_error_boards
 365         self._show_config = show_config
 366         self._show_environment = show_environment
 367         self._filter_dtb_warnings = filter_dtb_warnings
 368         self._filter_migration_warnings = filter_migration_warnings
 369
 370     def _AddTimestamp(self):
 371         """Add a new timestamp to the list and record the build period.
 372
 373         The build period is the length of time taken to perform a single
 374         build (one board, one commit).
 375         """
 376         now = datetime.now()
 377         self._timestamps.append(now)
 378         count = len(self._timestamps)
 379         delta = self._timestamps[-1] - self._timestamps[0]
 380         seconds = delta.total_seconds()
 381
 382         # If we have enough data, estimate build period (time taken for a
 383         # single build) and therefore completion time.
 384         if count > 1 and self._next_delay_update < now:
 385             self._next_delay_update = now + timedelta(seconds=2)
 386             if seconds > 0:
 387                 self._build_period = float(seconds) / count
 388                 todo = self.count - self.upto
 389                 self._complete_delay = timedelta(microseconds=
 390                         self._build_period * todo * 1000000)
 391                 # Round it
 392                 self._complete_delay -= timedelta(
 393                         microseconds=self._complete_delay.microseconds)
 394
 395         if seconds > 60:
 396             self._timestamps.popleft()
 397             count -= 1
 398
 399     def SelectCommit(self, commit, checkout=True):
 400         """Checkout the selected commit for this build
 401         """
 402         self.commit = commit
 403         if checkout and self.checkout:
 404             gitutil.Checkout(commit.hash)
 405
 406     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 407         """Run make
 408
 409         Args:
 410             commit: Commit object that is being built
 411             brd: Board object that is being built
 412             stage: Stage that we are at (mrproper, config, build)
 413             cwd: Directory where make should be run
 414             args: Arguments to pass to make
 415             kwargs: Arguments to pass to command.RunPipe()
 416         """
 417         cmd = [self.gnu_make] + list(args)
 418         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 419                 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
 420         if self.verbose_build:
 421             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 422             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 423         return result
 424
 425     def ProcessResult(self, result):
 426         """Process the result of a build, showing progress information
 427
 428         Args:
 429             result: A CommandResult object, which indicates the result for
 430                     a single build
 431         """
 432         col = terminal.Color()
 433         if result:
 434             target = result.brd.target
 435
 436             self.upto += 1
 437             if result.return_code != 0:
 438                 self.fail += 1
 439             elif result.stderr:
 440                 self.warned += 1
 441             if result.already_done:
 442                 self.already_done += 1
 443             if self._verbose:
 444                 terminal.PrintClear()
 445                 boards_selected = {target : result.brd}
 446                 self.ResetResultSummary(boards_selected)
 447                 self.ProduceResultSummary(result.commit_upto, self.commits,
 448                                           boards_selected)
 449         else:
 450             target = '(starting)'
 451
 452         # Display separate counts for ok, warned and fail
 453         ok = self.upto - self.warned - self.fail
 454         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 455         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 456         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 457
 458         line += ' /%-5d  ' % self.count
 459         remaining = self.count - self.upto
 460         if remaining:
 461             line += self.col.Color(self.col.MAGENTA, ' -%-5d  ' % remaining)
 462         else:
 463             line += ' ' * 8
 464
 465         # Add our current completion time estimate
 466         self._AddTimestamp()
 467         if self._complete_delay:
 468             line += '%s  : ' % self._complete_delay
 469
 470         line += target
 471         terminal.PrintClear()
 472         Print(line, newline=False, limit_to_line=True)
 473
 474     def _GetOutputDir(self, commit_upto):
 475         """Get the name of the output directory for a commit number
 476
 477         The output directory is typically .../<branch>/<commit>.
 478
 479         Args:
 480             commit_upto: Commit number to use (0..self.count-1)
 481         """
 482         if self.work_in_output:
 483             return self._working_dir
 484
 485         commit_dir = None
 486         if self.commits:
 487             commit = self.commits[commit_upto]
 488             subject = commit.subject.translate(trans_valid_chars)
 489             # See _GetOutputSpaceRemovals() which parses this name
 490             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
 491                     self.commit_count, commit.hash, subject[:20]))
 492         elif not self.no_subdirs:
 493             commit_dir = 'current'
 494         if not commit_dir:
 495             return self.base_dir
 496         return os.path.join(self.base_dir, commit_dir)
 497
 498     def GetBuildDir(self, commit_upto, target):
 499         """Get the name of the build directory for a commit number
 500
 501         The build directory is typically .../<branch>/<commit>/<target>.
 502
 503         Args:
 504             commit_upto: Commit number to use (0..self.count-1)
 505             target: Target name
 506         """
 507         output_dir = self._GetOutputDir(commit_upto)
 508         if self.work_in_output:
 509             return output_dir
 510         return os.path.join(output_dir, target)
 511
 512     def GetDoneFile(self, commit_upto, target):
 513         """Get the name of the done file for a commit number
 514
 515         Args:
 516             commit_upto: Commit number to use (0..self.count-1)
 517             target: Target name
 518         """
 519         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 520
 521     def GetSizesFile(self, commit_upto, target):
 522         """Get the name of the sizes file for a commit number
 523
 524         Args:
 525             commit_upto: Commit number to use (0..self.count-1)
 526             target: Target name
 527         """
 528         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 529
 530     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 531         """Get the name of the funcsizes file for a commit number and ELF file
 532
 533         Args:
 534             commit_upto: Commit number to use (0..self.count-1)
 535             target: Target name
 536             elf_fname: Filename of elf image
 537         """
 538         return os.path.join(self.GetBuildDir(commit_upto, target),
 539                             '%s.sizes' % elf_fname.replace('/', '-'))
 540
 541     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 542         """Get the name of the objdump file for a commit number and ELF file
 543
 544         Args:
 545             commit_upto: Commit number to use (0..self.count-1)
 546             target: Target name
 547             elf_fname: Filename of elf image
 548         """
 549         return os.path.join(self.GetBuildDir(commit_upto, target),
 550                             '%s.objdump' % elf_fname.replace('/', '-'))
 551
 552     def GetErrFile(self, commit_upto, target):
 553         """Get the name of the err file for a commit number
 554
 555         Args:
 556             commit_upto: Commit number to use (0..self.count-1)
 557             target: Target name
 558         """
 559         output_dir = self.GetBuildDir(commit_upto, target)
 560         return os.path.join(output_dir, 'err')
 561
 562     def FilterErrors(self, lines):
 563         """Filter out errors in which we have no interest
 564
 565         We should probably use map().
 566
 567         Args:
 568             lines: List of error lines, each a string
 569         Returns:
 570             New list with only interesting lines included
 571         """
 572         out_lines = []
 573         if self._filter_migration_warnings:
 574             text = '\n'.join(lines)
 575             text = self._re_migration_warning.sub('', text)
 576             lines = text.splitlines()
 577         for line in lines:
 578             if self.re_make_err.search(line):
 579                 continue
 580             if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
 581                 continue
 582             out_lines.append(line)
 583         return out_lines
 584
 585     def ReadFuncSizes(self, fname, fd):
 586         """Read function sizes from the output of 'nm'
 587
 588         Args:
 589             fd: File containing data to read
 590             fname: Filename we are reading from (just for errors)
 591
 592         Returns:
 593             Dictionary containing size of each function in bytes, indexed by
 594             function name.
 595         """
 596         sym = {}
 597         for line in fd.readlines():
 598             try:
 599                 if line.strip():
 600                     size, type, name = line[:-1].split()
 601             except:
 602                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 603                 continue
 604             if type in 'tTdDbB':
 605                 # function names begin with '.' on 64-bit powerpc
 606                 if '.' in name[1:]:
 607                     name = 'static.' + name.split('.')[0]
 608                 sym[name] = sym.get(name, 0) + int(size, 16)
 609         return sym
 610
 611     def _ProcessConfig(self, fname):
 612         """Read in a .config, autoconf.mk or autoconf.h file
 613
 614         This function handles all config file types. It ignores comments and
 615         any #defines which don't start with CONFIG_.
 616
 617         Args:
 618             fname: Filename to read
 619
 620         Returns:
 621             Dictionary:
 622                 key: Config name (e.g. CONFIG_DM)
 623                 value: Config value (e.g. 1)
 624         """
 625         config = {}
 626         if os.path.exists(fname):
 627             with open(fname) as fd:
 628                 for line in fd:
 629                     line = line.strip()
 630                     if line.startswith('#define'):
 631                         values = line[8:].split(' ', 1)
 632                         if len(values) > 1:
 633                             key, value = values
 634                         else:
 635                             key = values[0]
 636                             value = '1' if self.squash_config_y else ''
 637                         if not key.startswith('CONFIG_'):
 638                             continue
 639                     elif not line or line[0] in ['#', '*', '/']:
 640                         continue
 641                     else:
 642                         key, value = line.split('=', 1)
 643                     if self.squash_config_y and value == 'y':
 644                         value = '1'
 645                     config[key] = value
 646         return config
 647
 648     def _ProcessEnvironment(self, fname):
 649         """Read in a uboot.env file
 650
 651         This function reads in environment variables from a file.
 652
 653         Args:
 654             fname: Filename to read
 655
 656         Returns:
 657             Dictionary:
 658                 key: environment variable (e.g. bootlimit)
 659                 value: value of environment variable (e.g. 1)
 660         """
 661         environment = {}
 662         if os.path.exists(fname):
 663             with open(fname) as fd:
 664                 for line in fd.read().split('\0'):
 665                     try:
 666                         key, value = line.split('=', 1)
 667                         environment[key] = value
 668                     except ValueError:
 669                         # ignore lines we can't parse
 670                         pass
 671         return environment
 672
 673     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 674                         read_config, read_environment):
 675         """Work out the outcome of a build.
 676
 677         Args:
 678             commit_upto: Commit number to check (0..n-1)
 679             target: Target board to check
 680             read_func_sizes: True to read function size information
 681             read_config: True to read .config and autoconf.h files
 682             read_environment: True to read uboot.env files
 683
 684         Returns:
 685             Outcome object
 686         """
 687         done_file = self.GetDoneFile(commit_upto, target)
 688         sizes_file = self.GetSizesFile(commit_upto, target)
 689         sizes = {}
 690         func_sizes = {}
 691         config = {}
 692         environment = {}
 693         if os.path.exists(done_file):
 694             with open(done_file, 'r') as fd:
 695                 try:
 696                     return_code = int(fd.readline())
 697                 except ValueError:
 698                     # The file may be empty due to running out of disk space.
 699                     # Try a rebuild
 700                     return_code = 1
 701                 err_lines = []
 702                 err_file = self.GetErrFile(commit_upto, target)
 703                 if os.path.exists(err_file):
 704                     with open(err_file, 'r') as fd:
 705                         err_lines = self.FilterErrors(fd.readlines())
 706
 707                 # Decide whether the build was ok, failed or created warnings
 708                 if return_code:
 709                     rc = OUTCOME_ERROR
 710                 elif len(err_lines):
 711                     rc = OUTCOME_WARNING
 712                 else:
 713                     rc = OUTCOME_OK
 714
 715                 # Convert size information to our simple format
 716                 if os.path.exists(sizes_file):
 717                     with open(sizes_file, 'r') as fd:
 718                         for line in fd.readlines():
 719                             values = line.split()
 720                             rodata = 0
 721                             if len(values) > 6:
 722                                 rodata = int(values[6], 16)
 723                             size_dict = {
 724                                 'all' : int(values[0]) + int(values[1]) +
 725                                         int(values[2]),
 726                                 'text' : int(values[0]) - rodata,
 727                                 'data' : int(values[1]),
 728                                 'bss' : int(values[2]),
 729                                 'rodata' : rodata,
 730                             }
 731                             sizes[values[5]] = size_dict
 732
 733             if read_func_sizes:
 734                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 735                 for fname in glob.glob(pattern):
 736                     with open(fname, 'r') as fd:
 737                         dict_name = os.path.basename(fname).replace('.sizes',
 738                                                                     '')
 739                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 740
 741             if read_config:
 742                 output_dir = self.GetBuildDir(commit_upto, target)
 743                 for name in self.config_filenames:
 744                     fname = os.path.join(output_dir, name)
 745                     config[name] = self._ProcessConfig(fname)
 746
 747             if read_environment:
 748                 output_dir = self.GetBuildDir(commit_upto, target)
 749                 fname = os.path.join(output_dir, 'uboot.env')
 750                 environment = self._ProcessEnvironment(fname)
 751
 752             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 753                                    environment)
 754
 755         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 756
 757     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 758                          read_config, read_environment):
 759         """Calculate a summary of the results of building a commit.
 760
 761         Args:
 762             board_selected: Dict containing boards to summarise
 763             commit_upto: Commit number to summarize (0..self.count-1)
 764             read_func_sizes: True to read function size information
 765             read_config: True to read .config and autoconf.h files
 766             read_environment: True to read uboot.env files
 767
 768         Returns:
 769             Tuple:
 770                 Dict containing boards which passed building this commit.
 771                     keyed by board.target
 772                 List containing a summary of error lines
 773                 Dict keyed by error line, containing a list of the Board
 774                     objects with that error
 775                 List containing a summary of warning lines
 776                 Dict keyed by error line, containing a list of the Board
 777                     objects with that warning
 778                 Dictionary keyed by board.target. Each value is a dictionary:
 779                     key: filename - e.g. '.config'
 780                     value is itself a dictionary:
 781                         key: config name
 782                         value: config value
 783                 Dictionary keyed by board.target. Each value is a dictionary:
 784                     key: environment variable
 785                     value: value of environment variable
 786         """
 787         def AddLine(lines_summary, lines_boards, line, board):
 788             line = line.rstrip()
 789             if line in lines_boards:
 790                 lines_boards[line].append(board)
 791             else:
 792                 lines_boards[line] = [board]
 793                 lines_summary.append(line)
 794
 795         board_dict = {}
 796         err_lines_summary = []
 797         err_lines_boards = {}
 798         warn_lines_summary = []
 799         warn_lines_boards = {}
 800         config = {}
 801         environment = {}
 802
 803         for board in boards_selected.values():
 804             outcome = self.GetBuildOutcome(commit_upto, board.target,
 805                                            read_func_sizes, read_config,
 806                                            read_environment)
 807             board_dict[board.target] = outcome
 808             last_func = None
 809             last_was_warning = False
 810             for line in outcome.err_lines:
 811                 if line:
 812                     if (self._re_function.match(line) or
 813                             self._re_files.match(line)):
 814                         last_func = line
 815                     else:
 816                         is_warning = (self._re_warning.match(line) or
 817                                       self._re_dtb_warning.match(line))
 818                         is_note = self._re_note.match(line)
 819                         if is_warning or (last_was_warning and is_note):
 820                             if last_func:
 821                                 AddLine(warn_lines_summary, warn_lines_boards,
 822                                         last_func, board)
 823                             AddLine(warn_lines_summary, warn_lines_boards,
 824                                     line, board)
 825                         else:
 826                             if last_func:
 827                                 AddLine(err_lines_summary, err_lines_boards,
 828                                         last_func, board)
 829                             AddLine(err_lines_summary, err_lines_boards,
 830                                     line, board)
 831                         last_was_warning = is_warning
 832                         last_func = None
 833             tconfig = Config(self.config_filenames, board.target)
 834             for fname in self.config_filenames:
 835                 if outcome.config:
 836                     for key, value in outcome.config[fname].items():
 837                         tconfig.Add(fname, key, value)
 838             config[board.target] = tconfig
 839
 840             tenvironment = Environment(board.target)
 841             if outcome.environment:
 842                 for key, value in outcome.environment.items():
 843                     tenvironment.Add(key, value)
 844             environment[board.target] = tenvironment
 845
 846         return (board_dict, err_lines_summary, err_lines_boards,
 847                 warn_lines_summary, warn_lines_boards, config, environment)
 848
 849     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 850         """Add an output to our list of outcomes for each architecture
 851
 852         This simple function adds failing boards (changes) to the
 853         relevant architecture string, so we can print the results out
 854         sorted by architecture.
 855
 856         Args:
 857              board_dict: Dict containing all boards
 858              arch_list: Dict keyed by arch name. Value is a string containing
 859                     a list of board names which failed for that arch.
 860              changes: List of boards to add to arch_list
 861              color: terminal.Colour object
 862         """
 863         done_arch = {}
 864         for target in changes:
 865             if target in board_dict:
 866                 arch = board_dict[target].arch
 867             else:
 868                 arch = 'unknown'
 869             str = self.col.Color(color, ' ' + target)
 870             if not arch in done_arch:
 871                 str = ' %s  %s' % (self.col.Color(color, char), str)
 872                 done_arch[arch] = True
 873             if not arch in arch_list:
 874                 arch_list[arch] = str
 875             else:
 876                 arch_list[arch] += str
 877
 878
 879     def ColourNum(self, num):
 880         color = self.col.RED if num > 0 else self.col.GREEN
 881         if num == 0:
 882             return '0'
 883         return self.col.Color(color, str(num))
 884
 885     def ResetResultSummary(self, board_selected):
 886         """Reset the results summary ready for use.
 887
 888         Set up the base board list to be all those selected, and set the
 889         error lines to empty.
 890
 891         Following this, calls to PrintResultSummary() will use this
 892         information to work out what has changed.
 893
 894         Args:
 895             board_selected: Dict containing boards to summarise, keyed by
 896                 board.target
 897         """
 898         self._base_board_dict = {}
 899         for board in board_selected:
 900             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 901                                                            {})
 902         self._base_err_lines = []
 903         self._base_warn_lines = []
 904         self._base_err_line_boards = {}
 905         self._base_warn_line_boards = {}
 906         self._base_config = None
 907         self._base_environment = None
 908
 909     def PrintFuncSizeDetail(self, fname, old, new):
 910         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 911         delta, common = [], {}
 912
 913         for a in old:
 914             if a in new:
 915                 common[a] = 1
 916
 917         for name in old:
 918             if name not in common:
 919                 remove += 1
 920                 down += old[name]
 921                 delta.append([-old[name], name])
 922
 923         for name in new:
 924             if name not in common:
 925                 add += 1
 926                 up += new[name]
 927                 delta.append([new[name], name])
 928
 929         for name in common:
 930                 diff = new.get(name, 0) - old.get(name, 0)
 931                 if diff > 0:
 932                     grow, up = grow + 1, up + diff
 933                 elif diff < 0:
 934                     shrink, down = shrink + 1, down - diff
 935                 delta.append([diff, name])
 936
 937         delta.sort()
 938         delta.reverse()
 939
 940         args = [add, -remove, grow, -shrink, up, -down, up - down]
 941         if max(args) == 0 and min(args) == 0:
 942             return
 943         args = [self.ColourNum(x) for x in args]
 944         indent = ' ' * 15
 945         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
 946               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
 947         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
 948                                          'delta'))
 949         for diff, name in delta:
 950             if diff:
 951                 color = self.col.RED if diff > 0 else self.col.GREEN
 952                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
 953                         old.get(name, '-'), new.get(name,'-'), diff)
 954                 Print(msg, colour=color)
 955
 956
 957     def PrintSizeDetail(self, target_list, show_bloat):
 958         """Show details size information for each board
 959
 960         Args:
 961             target_list: List of targets, each a dict containing:
 962                     'target': Target name
 963                     'total_diff': Total difference in bytes across all areas
 964                     <part_name>: Difference for that part
 965             show_bloat: Show detail for each function
 966         """
 967         targets_by_diff = sorted(target_list, reverse=True,
 968         key=lambda x: x['_total_diff'])
 969         for result in targets_by_diff:
 970             printed_target = False
 971             for name in sorted(result):
 972                 diff = result[name]
 973                 if name.startswith('_'):
 974                     continue
 975                 if diff != 0:
 976                     color = self.col.RED if diff > 0 else self.col.GREEN
 977                 msg = ' %s %+d' % (name, diff)
 978                 if not printed_target:
 979                     Print('%10s  %-15s:' % ('', result['_target']),
 980                           newline=False)
 981                     printed_target = True
 982                 Print(msg, colour=color, newline=False)
 983             if printed_target:
 984                 Print()
 985                 if show_bloat:
 986                     target = result['_target']
 987                     outcome = result['_outcome']
 988                     base_outcome = self._base_board_dict[target]
 989                     for fname in outcome.func_sizes:
 990                         self.PrintFuncSizeDetail(fname,
 991                                                  base_outcome.func_sizes[fname],
 992                                                  outcome.func_sizes[fname])
 993
 994
 995     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
 996                          show_bloat):
 997         """Print a summary of image sizes broken down by section.
 998
 999         The summary takes the form of one line per architecture. The
1000         line contains deltas for each of the sections (+ means the section
1001         got bigger, - means smaller). The numbers are the average number
1002         of bytes that a board in this section increased by.
1003
1004         For example:
1005            powerpc: (622 boards)   text -0.0
1006           arm: (285 boards)   text -0.0
1007           nds32: (3 boards)   text -8.0
1008
1009         Args:
1010             board_selected: Dict containing boards to summarise, keyed by
1011                 board.target
1012             board_dict: Dict containing boards for which we built this
1013                 commit, keyed by board.target. The value is an Outcome object.
1014             show_detail: Show size delta detail for each board
1015             show_bloat: Show detail for each function
1016         """
1017         arch_list = {}
1018         arch_count = {}
1019
1020         # Calculate changes in size for different image parts
1021         # The previous sizes are in Board.sizes, for each board
1022         for target in board_dict:
1023             if target not in board_selected:
1024                 continue
1025             base_sizes = self._base_board_dict[target].sizes
1026             outcome = board_dict[target]
1027             sizes = outcome.sizes
1028
1029             # Loop through the list of images, creating a dict of size
1030             # changes for each image/part. We end up with something like
1031             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1032             # which means that U-Boot data increased by 5 bytes and SPL
1033             # text decreased by 4.
1034             err = {'_target' : target}
1035             for image in sizes:
1036                 if image in base_sizes:
1037                     base_image = base_sizes[image]
1038                     # Loop through the text, data, bss parts
1039                     for part in sorted(sizes[image]):
1040                         diff = sizes[image][part] - base_image[part]
1041                         col = None
1042                         if diff:
1043                             if image == 'u-boot':
1044                                 name = part
1045                             else:
1046                                 name = image + ':' + part
1047                             err[name] = diff
1048             arch = board_selected[target].arch
1049             if not arch in arch_count:
1050                 arch_count[arch] = 1
1051             else:
1052                 arch_count[arch] += 1
1053             if not sizes:
1054                 pass    # Only add to our list when we have some stats
1055             elif not arch in arch_list:
1056                 arch_list[arch] = [err]
1057             else:
1058                 arch_list[arch].append(err)
1059
1060         # We now have a list of image size changes sorted by arch
1061         # Print out a summary of these
1062         for arch, target_list in arch_list.items():
1063             # Get total difference for each type
1064             totals = {}
1065             for result in target_list:
1066                 total = 0
1067                 for name, diff in result.items():
1068                     if name.startswith('_'):
1069                         continue
1070                     total += diff
1071                     if name in totals:
1072                         totals[name] += diff
1073                     else:
1074                         totals[name] = diff
1075                 result['_total_diff'] = total
1076                 result['_outcome'] = board_dict[result['_target']]
1077
1078             count = len(target_list)
1079             printed_arch = False
1080             for name in sorted(totals):
1081                 diff = totals[name]
1082                 if diff:
1083                     # Display the average difference in this name for this
1084                     # architecture
1085                     avg_diff = float(diff) / count
1086                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1087                     msg = ' %s %+1.1f' % (name, avg_diff)
1088                     if not printed_arch:
1089                         Print('%10s: (for %d/%d boards)' % (arch, count,
1090                               arch_count[arch]), newline=False)
1091                         printed_arch = True
1092                     Print(msg, colour=color, newline=False)
1093
1094             if printed_arch:
1095                 Print()
1096                 if show_detail:
1097                     self.PrintSizeDetail(target_list, show_bloat)
1098
1099
1100     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1101                            err_line_boards, warn_lines, warn_line_boards,
1102                            config, environment, show_sizes, show_detail,
1103                            show_bloat, show_config, show_environment):
1104         """Compare results with the base results and display delta.
1105
1106         Only boards mentioned in board_selected will be considered. This
1107         function is intended to be called repeatedly with the results of
1108         each commit. It therefore shows a 'diff' between what it saw in
1109         the last call and what it sees now.
1110
1111         Args:
1112             board_selected: Dict containing boards to summarise, keyed by
1113                 board.target
1114             board_dict: Dict containing boards for which we built this
1115                 commit, keyed by board.target. The value is an Outcome object.
1116             err_lines: A list of errors for this commit, or [] if there is
1117                 none, or we don't want to print errors
1118             err_line_boards: Dict keyed by error line, containing a list of
1119                 the Board objects with that error
1120             warn_lines: A list of warnings for this commit, or [] if there is
1121                 none, or we don't want to print errors
1122             warn_line_boards: Dict keyed by warning line, containing a list of
1123                 the Board objects with that warning
1124             config: Dictionary keyed by filename - e.g. '.config'. Each
1125                     value is itself a dictionary:
1126                         key: config name
1127                         value: config value
1128             environment: Dictionary keyed by environment variable, Each
1129                      value is the value of environment variable.
1130             show_sizes: Show image size deltas
1131             show_detail: Show size delta detail for each board if show_sizes
1132             show_bloat: Show detail for each function
1133             show_config: Show config changes
1134             show_environment: Show environment changes
1135         """
1136         def _BoardList(line, line_boards):
1137             """Helper function to get a line of boards containing a line
1138
1139             Args:
1140                 line: Error line to search for
1141                 line_boards: boards to search, each a Board
1142             Return:
1143                 List of boards with that error line, or [] if the user has not
1144                     requested such a list
1145             """
1146             boards = []
1147             board_set = set()
1148             if self._list_error_boards:
1149                 for board in line_boards[line]:
1150                     if not board in board_set:
1151                         boards.append(board)
1152                         board_set.add(board)
1153             return boards
1154
1155         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1156                             char):
1157             """Calculate the required output based on changes in errors
1158
1159             Args:
1160                 base_lines: List of errors/warnings for previous commit
1161                 base_line_boards: Dict keyed by error line, containing a list
1162                     of the Board objects with that error in the previous commit
1163                 lines: List of errors/warning for this commit, each a str
1164                 line_boards: Dict keyed by error line, containing a list
1165                     of the Board objects with that error in this commit
1166                 char: Character representing error ('') or warning ('w'). The
1167                     broken ('+') or fixed ('-') characters are added in this
1168                     function
1169
1170             Returns:
1171                 Tuple
1172                     List of ErrLine objects for 'better' lines
1173                     List of ErrLine objects for 'worse' lines
1174             """
1175             better_lines = []
1176             worse_lines = []
1177             for line in lines:
1178                 if line not in base_lines:
1179                     errline = ErrLine(char + '+', _BoardList(line, line_boards),
1180                                       line)
1181                     worse_lines.append(errline)
1182             for line in base_lines:
1183                 if line not in lines:
1184                     errline = ErrLine(char + '-',
1185                                       _BoardList(line, base_line_boards), line)
1186                     better_lines.append(errline)
1187             return better_lines, worse_lines
1188
1189         def _CalcConfig(delta, name, config):
1190             """Calculate configuration changes
1191
1192             Args:
1193                 delta: Type of the delta, e.g. '+'
1194                 name: name of the file which changed (e.g. .config)
1195                 config: configuration change dictionary
1196                     key: config name
1197                     value: config value
1198             Returns:
1199                 String containing the configuration changes which can be
1200                     printed
1201             """
1202             out = ''
1203             for key in sorted(config.keys()):
1204                 out += '%s=%s ' % (key, config[key])
1205             return '%s %s: %s' % (delta, name, out)
1206
1207         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1208             """Add changes in configuration to a list
1209
1210             Args:
1211                 lines: list to add to
1212                 name: config file name
1213                 config_plus: configurations added, dictionary
1214                     key: config name
1215                     value: config value
1216                 config_minus: configurations removed, dictionary
1217                     key: config name
1218                     value: config value
1219                 config_change: configurations changed, dictionary
1220                     key: config name
1221                     value: config value
1222             """
1223             if config_plus:
1224                 lines.append(_CalcConfig('+', name, config_plus))
1225             if config_minus:
1226                 lines.append(_CalcConfig('-', name, config_minus))
1227             if config_change:
1228                 lines.append(_CalcConfig('c', name, config_change))
1229
1230         def _OutputConfigInfo(lines):
1231             for line in lines:
1232                 if not line:
1233                     continue
1234                 if line[0] == '+':
1235                     col = self.col.GREEN
1236                 elif line[0] == '-':
1237                     col = self.col.RED
1238                 elif line[0] == 'c':
1239                     col = self.col.YELLOW
1240                 Print('   ' + line, newline=True, colour=col)
1241
1242         def _OutputErrLines(err_lines, colour):
1243             """Output the line of error/warning lines, if not empty
1244
1245             Also increments self._error_lines if err_lines not empty
1246
1247             Args:
1248                 err_lines: List of ErrLine objects, each an error or warning
1249                     line, possibly including a list of boards with that
1250                     error/warning
1251                 colour: Colour to use for output
1252             """
1253             if err_lines:
1254                 out_list = []
1255                 for line in err_lines:
1256                     boards = ''
1257                     names = [board.target for board in line.boards]
1258                     board_str = ' '.join(names) if names else ''
1259                     if board_str:
1260                         out = self.col.Color(colour, line.char + '(')
1261                         out += self.col.Color(self.col.MAGENTA, board_str,
1262                                               bright=False)
1263                         out += self.col.Color(colour, ') %s' % line.errline)
1264                     else:
1265                         out = self.col.Color(colour, line.char + line.errline)
1266                     out_list.append(out)
1267                 Print('\n'.join(out_list))
1268                 self._error_lines += 1
1269
1270
1271         ok_boards = []      # List of boards fixed since last commit
1272         warn_boards = []    # List of boards with warnings since last commit
1273         err_boards = []     # List of new broken boards since last commit
1274         new_boards = []     # List of boards that didn't exist last time
1275         unknown_boards = [] # List of boards that were not built
1276
1277         for target in board_dict:
1278             if target not in board_selected:
1279                 continue
1280
1281             # If the board was built last time, add its outcome to a list
1282             if target in self._base_board_dict:
1283                 base_outcome = self._base_board_dict[target].rc
1284                 outcome = board_dict[target]
1285                 if outcome.rc == OUTCOME_UNKNOWN:
1286                     unknown_boards.append(target)
1287                 elif outcome.rc < base_outcome:
1288                     if outcome.rc == OUTCOME_WARNING:
1289                         warn_boards.append(target)
1290                     else:
1291                         ok_boards.append(target)
1292                 elif outcome.rc > base_outcome:
1293                     if outcome.rc == OUTCOME_WARNING:
1294                         warn_boards.append(target)
1295                     else:
1296                         err_boards.append(target)
1297             else:
1298                 new_boards.append(target)
1299
1300         # Get a list of errors and warnings that have appeared, and disappeared
1301         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1302                 self._base_err_line_boards, err_lines, err_line_boards, '')
1303         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1304                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1305
1306         # Display results by arch
1307         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1308                 worse_err, better_err, worse_warn, better_warn)):
1309             arch_list = {}
1310             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1311                     self.col.GREEN)
1312             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1313                     self.col.YELLOW)
1314             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1315                     self.col.RED)
1316             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1317             if self._show_unknown:
1318                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1319                         self.col.MAGENTA)
1320             for arch, target_list in arch_list.items():
1321                 Print('%10s: %s' % (arch, target_list))
1322                 self._error_lines += 1
1323             _OutputErrLines(better_err, colour=self.col.GREEN)
1324             _OutputErrLines(worse_err, colour=self.col.RED)
1325             _OutputErrLines(better_warn, colour=self.col.CYAN)
1326             _OutputErrLines(worse_warn, colour=self.col.YELLOW)
1327
1328         if show_sizes:
1329             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1330                                   show_bloat)
1331
1332         if show_environment and self._base_environment:
1333             lines = []
1334
1335             for target in board_dict:
1336                 if target not in board_selected:
1337                     continue
1338
1339                 tbase = self._base_environment[target]
1340                 tenvironment = environment[target]
1341                 environment_plus = {}
1342                 environment_minus = {}
1343                 environment_change = {}
1344                 base = tbase.environment
1345                 for key, value in tenvironment.environment.items():
1346                     if key not in base:
1347                         environment_plus[key] = value
1348                 for key, value in base.items():
1349                     if key not in tenvironment.environment:
1350                         environment_minus[key] = value
1351                 for key, value in base.items():
1352                     new_value = tenvironment.environment.get(key)
1353                     if new_value and value != new_value:
1354                         desc = '%s -> %s' % (value, new_value)
1355                         environment_change[key] = desc
1356
1357                 _AddConfig(lines, target, environment_plus, environment_minus,
1358                            environment_change)
1359
1360             _OutputConfigInfo(lines)
1361
1362         if show_config and self._base_config:
1363             summary = {}
1364             arch_config_plus = {}
1365             arch_config_minus = {}
1366             arch_config_change = {}
1367             arch_list = []
1368
1369             for target in board_dict:
1370                 if target not in board_selected:
1371                     continue
1372                 arch = board_selected[target].arch
1373                 if arch not in arch_list:
1374                     arch_list.append(arch)
1375
1376             for arch in arch_list:
1377                 arch_config_plus[arch] = {}
1378                 arch_config_minus[arch] = {}
1379                 arch_config_change[arch] = {}
1380                 for name in self.config_filenames:
1381                     arch_config_plus[arch][name] = {}
1382                     arch_config_minus[arch][name] = {}
1383                     arch_config_change[arch][name] = {}
1384
1385             for target in board_dict:
1386                 if target not in board_selected:
1387                     continue
1388
1389                 arch = board_selected[target].arch
1390
1391                 all_config_plus = {}
1392                 all_config_minus = {}
1393                 all_config_change = {}
1394                 tbase = self._base_config[target]
1395                 tconfig = config[target]
1396                 lines = []
1397                 for name in self.config_filenames:
1398                     if not tconfig.config[name]:
1399                         continue
1400                     config_plus = {}
1401                     config_minus = {}
1402                     config_change = {}
1403                     base = tbase.config[name]
1404                     for key, value in tconfig.config[name].items():
1405                         if key not in base:
1406                             config_plus[key] = value
1407                             all_config_plus[key] = value
1408                     for key, value in base.items():
1409                         if key not in tconfig.config[name]:
1410                             config_minus[key] = value
1411                             all_config_minus[key] = value
1412                     for key, value in base.items():
1413                         new_value = tconfig.config.get(key)
1414                         if new_value and value != new_value:
1415                             desc = '%s -> %s' % (value, new_value)
1416                             config_change[key] = desc
1417                             all_config_change[key] = desc
1418
1419                     arch_config_plus[arch][name].update(config_plus)
1420                     arch_config_minus[arch][name].update(config_minus)
1421                     arch_config_change[arch][name].update(config_change)
1422
1423                     _AddConfig(lines, name, config_plus, config_minus,
1424                                config_change)
1425                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1426                            all_config_change)
1427                 summary[target] = '\n'.join(lines)
1428
1429             lines_by_target = {}
1430             for target, lines in summary.items():
1431                 if lines in lines_by_target:
1432                     lines_by_target[lines].append(target)
1433                 else:
1434                     lines_by_target[lines] = [target]
1435
1436             for arch in arch_list:
1437                 lines = []
1438                 all_plus = {}
1439                 all_minus = {}
1440                 all_change = {}
1441                 for name in self.config_filenames:
1442                     all_plus.update(arch_config_plus[arch][name])
1443                     all_minus.update(arch_config_minus[arch][name])
1444                     all_change.update(arch_config_change[arch][name])
1445                     _AddConfig(lines, name, arch_config_plus[arch][name],
1446                                arch_config_minus[arch][name],
1447                                arch_config_change[arch][name])
1448                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1449                 #arch_summary[target] = '\n'.join(lines)
1450                 if lines:
1451                     Print('%s:' % arch)
1452                     _OutputConfigInfo(lines)
1453
1454             for lines, targets in lines_by_target.items():
1455                 if not lines:
1456                     continue
1457                 Print('%s :' % ' '.join(sorted(targets)))
1458                 _OutputConfigInfo(lines.split('\n'))
1459
1460
1461         # Save our updated information for the next call to this function
1462         self._base_board_dict = board_dict
1463         self._base_err_lines = err_lines
1464         self._base_warn_lines = warn_lines
1465         self._base_err_line_boards = err_line_boards
1466         self._base_warn_line_boards = warn_line_boards
1467         self._base_config = config
1468         self._base_environment = environment
1469
1470         # Get a list of boards that did not get built, if needed
1471         not_built = []
1472         for board in board_selected:
1473             if not board in board_dict:
1474                 not_built.append(board)
1475         if not_built:
1476             Print("Boards not built (%d): %s" % (len(not_built),
1477                   ', '.join(not_built)))
1478
1479     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1480             (board_dict, err_lines, err_line_boards, warn_lines,
1481              warn_line_boards, config, environment) = self.GetResultSummary(
1482                     board_selected, commit_upto,
1483                     read_func_sizes=self._show_bloat,
1484                     read_config=self._show_config,
1485                     read_environment=self._show_environment)
1486             if commits:
1487                 msg = '%02d: %s' % (commit_upto + 1,
1488                         commits[commit_upto].subject)
1489                 Print(msg, colour=self.col.BLUE)
1490             self.PrintResultSummary(board_selected, board_dict,
1491                     err_lines if self._show_errors else [], err_line_boards,
1492                     warn_lines if self._show_errors else [], warn_line_boards,
1493                     config, environment, self._show_sizes, self._show_detail,
1494                     self._show_bloat, self._show_config, self._show_environment)
1495
1496     def ShowSummary(self, commits, board_selected):
1497         """Show a build summary for U-Boot for a given board list.
1498
1499         Reset the result summary, then repeatedly call GetResultSummary on
1500         each commit's results, then display the differences we see.
1501
1502         Args:
1503             commit: Commit objects to summarise
1504             board_selected: Dict containing boards to summarise
1505         """
1506         self.commit_count = len(commits) if commits else 1
1507         self.commits = commits
1508         self.ResetResultSummary(board_selected)
1509         self._error_lines = 0
1510
1511         for commit_upto in range(0, self.commit_count, self._step):
1512             self.ProduceResultSummary(commit_upto, commits, board_selected)
1513         if not self._error_lines:
1514             Print('(no errors to report)', colour=self.col.GREEN)
1515
1516
1517     def SetupBuild(self, board_selected, commits):
1518         """Set up ready to start a build.
1519
1520         Args:
1521             board_selected: Selected boards to build
1522             commits: Selected commits to build
1523         """
1524         # First work out how many commits we will build
1525         count = (self.commit_count + self._step - 1) // self._step
1526         self.count = len(board_selected) * count
1527         self.upto = self.warned = self.fail = 0
1528         self._timestamps = collections.deque()
1529
1530     def GetThreadDir(self, thread_num):
1531         """Get the directory path to the working dir for a thread.
1532
1533         Args:
1534             thread_num: Number of thread to check.
1535         """
1536         if self.work_in_output:
1537             return self._working_dir
1538         return os.path.join(self._working_dir, '%02d' % thread_num)
1539
1540     def _PrepareThread(self, thread_num, setup_git):
1541         """Prepare the working directory for a thread.
1542
1543         This clones or fetches the repo into the thread's work directory.
1544
1545         Args:
1546             thread_num: Thread number (0, 1, ...)
1547             setup_git: True to set up a git repo clone
1548         """
1549         thread_dir = self.GetThreadDir(thread_num)
1550         builderthread.Mkdir(thread_dir)
1551         git_dir = os.path.join(thread_dir, '.git')
1552
1553         # Clone the repo if it doesn't already exist
1554         # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1555         # we have a private index but uses the origin repo's contents?
1556         if setup_git and self.git_dir:
1557             src_dir = os.path.abspath(self.git_dir)
1558             if os.path.exists(git_dir):
1559                 Print('\rFetching repo for thread %d' % thread_num,
1560                       newline=False)
1561                 gitutil.Fetch(git_dir, thread_dir)
1562                 terminal.PrintClear()
1563             else:
1564                 Print('\rCloning repo for thread %d' % thread_num,
1565                       newline=False)
1566                 gitutil.Clone(src_dir, thread_dir)
1567                 terminal.PrintClear()
1568
1569     def _PrepareWorkingSpace(self, max_threads, setup_git):
1570         """Prepare the working directory for use.
1571
1572         Set up the git repo for each thread.
1573
1574         Args:
1575             max_threads: Maximum number of threads we expect to need.
1576             setup_git: True to set up a git repo clone
1577         """
1578         builderthread.Mkdir(self._working_dir)
1579         for thread in range(max_threads):
1580             self._PrepareThread(thread, setup_git)
1581
1582     def _GetOutputSpaceRemovals(self):
1583         """Get the output directories ready to receive files.
1584
1585         Figure out what needs to be deleted in the output directory before it
1586         can be used. We only delete old buildman directories which have the
1587         expected name pattern. See _GetOutputDir().
1588
1589         Returns:
1590             List of full paths of directories to remove
1591         """
1592         if not self.commits:
1593             return
1594         dir_list = []
1595         for commit_upto in range(self.commit_count):
1596             dir_list.append(self._GetOutputDir(commit_upto))
1597
1598         to_remove = []
1599         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1600             if dirname not in dir_list:
1601                 leaf = dirname[len(self.base_dir) + 1:]
1602                 m =  re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1603                 if m:
1604                     to_remove.append(dirname)
1605         return to_remove
1606
1607     def _PrepareOutputSpace(self):
1608         """Get the output directories ready to receive files.
1609
1610         We delete any output directories which look like ones we need to
1611         create. Having left over directories is confusing when the user wants
1612         to check the output manually.
1613         """
1614         to_remove = self._GetOutputSpaceRemovals()
1615         if to_remove:
1616             Print('Removing %d old build directories...' % len(to_remove),
1617                   newline=False)
1618             for dirname in to_remove:
1619                 shutil.rmtree(dirname)
1620             terminal.PrintClear()
1621
1622     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1623         """Build all commits for a list of boards
1624
1625         Args:
1626             commits: List of commits to be build, each a Commit object
1627             boards_selected: Dict of selected boards, key is target name,
1628                     value is Board object
1629             keep_outputs: True to save build output files
1630             verbose: Display build results as they are completed
1631         Returns:
1632             Tuple containing:
1633                 - number of boards that failed to build
1634                 - number of boards that issued warnings
1635         """
1636         self.commit_count = len(commits) if commits else 1
1637         self.commits = commits
1638         self._verbose = verbose
1639
1640         self.ResetResultSummary(board_selected)
1641         builderthread.Mkdir(self.base_dir, parents = True)
1642         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1643                 commits is not None)
1644         self._PrepareOutputSpace()
1645         Print('\rStarting build...', newline=False)
1646         self.SetupBuild(board_selected, commits)
1647         self.ProcessResult(None)
1648
1649         # Create jobs to build all commits for each board
1650         for brd in board_selected.values():
1651             job = builderthread.BuilderJob()
1652             job.board = brd
1653             job.commits = commits
1654             job.keep_outputs = keep_outputs
1655             job.work_in_output = self.work_in_output
1656             job.step = self._step
1657             self.queue.put(job)
1658
1659         term = threading.Thread(target=self.queue.join)
1660         term.setDaemon(True)
1661         term.start()
1662         while term.isAlive():
1663             term.join(100)
1664
1665         # Wait until we have processed all output
1666         self.out_queue.join()
1667         Print()
1668
1669         msg = 'Completed: %d total built' % self.count
1670         if self.already_done:
1671            msg += ' (%d previously' % self.already_done
1672            if self.already_done != self.count:
1673                msg += ', %d newly' % (self.count - self.already_done)
1674            msg += ')'
1675         duration = datetime.now() - self._start_time
1676         if duration > timedelta(microseconds=1000000):
1677             if duration.microseconds >= 500000:
1678                 duration = duration + timedelta(seconds=1)
1679             duration = duration - timedelta(microseconds=duration.microseconds)
1680             msg += ', duration %s' % duration
1681         Print(msg)
1682
1683         return (self.fail, self.warned)