tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 import builderthread
  21 import command
  22 import gitutil
  23 import terminal
  24 from terminal import Print
  25 import toolchain
  26
  27 """
  28 Theory of Operation
  29
  30 Please see README for user documentation, and you should be familiar with
  31 that before trying to make sense of this.
  32
  33 Buildman works by keeping the machine as busy as possible, building different
  34 commits for different boards on multiple CPUs at once.
  35
  36 The source repo (self.git_dir) contains all the commits to be built. Each
  37 thread works on a single board at a time. It checks out the first commit,
  38 configures it for that board, then builds it. Then it checks out the next
  39 commit and builds it (typically without re-configuring). When it runs out
  40 of commits, it gets another job from the builder and starts again with that
  41 board.
  42
  43 Clearly the builder threads could work either way - they could check out a
  44 commit and then built it for all boards. Using separate directories for each
  45 commit/board pair they could leave their build product around afterwards
  46 also.
  47
  48 The intent behind building a single board for multiple commits, is to make
  49 use of incremental builds. Since each commit is built incrementally from
  50 the previous one, builds are faster. Reconfiguring for a different board
  51 removes all intermediate object files.
  52
  53 Many threads can be working at once, but each has its own working directory.
  54 When a thread finishes a build, it puts the output files into a result
  55 directory.
  56
  57 The base directory used by buildman is normally '../<branch>', i.e.
  58 a directory higher than the source repository and named after the branch
  59 being built.
  60
  61 Within the base directory, we have one subdirectory for each commit. Within
  62 that is one subdirectory for each board. Within that is the build output for
  63 that commit/board combination.
  64
  65 Buildman also create working directories for each thread, in a .bm-work/
  66 subdirectory in the base dir.
  67
  68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  69 'seaboard', and say that us-net has two commits. We will have directories
  70 like this:
  71
  72 us-net/             base directory
  73     01_of_02_g4ed4ebc_net--Add-tftp-speed-/
  74         sandbox/
  75             u-boot.bin
  76         seaboard/
  77             u-boot.bin
  78     02_of_02_g4ed4ebc_net--Check-tftp-comp/
  79         sandbox/
  80             u-boot.bin
  81         seaboard/
  82             u-boot.bin
  83     .bm-work/
  84         00/         working directory for thread 0 (contains source checkout)
  85             build/  build output
  86         01/         working directory for thread 1
  87             build/  build output
  88         ...
  89 u-boot/             source directory
  90     .git/           repository
  91 """
  92
  93 """Holds information about a particular error line we are outputing
  94
  95    char: Character representation: '+': error, '-': fixed error, 'w+': warning,
  96        'w-' = fixed warning
  97    boards: List of Board objects which have line in the error/warning output
  98    errline: The text of the error line
  99 """
 100 ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
 101
 102 # Possible build outcomes
 103 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 104
 105 # Translate a commit subject into a valid filename (and handle unicode)
 106 trans_valid_chars = str.maketrans('/: ', '---')
 107
 108 BASE_CONFIG_FILENAMES = [
 109     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 110 ]
 111
 112 EXTRA_CONFIG_FILENAMES = [
 113     '.config', '.config-spl', '.config-tpl',
 114     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 115     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 116 ]
 117
 118 class Config:
 119     """Holds information about configuration settings for a board."""
 120     def __init__(self, config_filename, target):
 121         self.target = target
 122         self.config = {}
 123         for fname in config_filename:
 124             self.config[fname] = {}
 125
 126     def Add(self, fname, key, value):
 127         self.config[fname][key] = value
 128
 129     def __hash__(self):
 130         val = 0
 131         for fname in self.config:
 132             for key, value in self.config[fname].items():
 133                 print(key, value)
 134                 val = val ^ hash(key) & hash(value)
 135         return val
 136
 137 class Environment:
 138     """Holds information about environment variables for a board."""
 139     def __init__(self, target):
 140         self.target = target
 141         self.environment = {}
 142
 143     def Add(self, key, value):
 144         self.environment[key] = value
 145
 146 class Builder:
 147     """Class for building U-Boot for a particular commit.
 148
 149     Public members: (many should ->private)
 150         already_done: Number of builds already completed
 151         base_dir: Base directory to use for builder
 152         checkout: True to check out source, False to skip that step.
 153             This is used for testing.
 154         col: terminal.Color() object
 155         count: Number of commits to build
 156         do_make: Method to call to invoke Make
 157         fail: Number of builds that failed due to error
 158         force_build: Force building even if a build already exists
 159         force_config_on_failure: If a commit fails for a board, disable
 160             incremental building for the next commit we build for that
 161             board, so that we will see all warnings/errors again.
 162         force_build_failures: If a previously-built build (i.e. built on
 163             a previous run of buildman) is marked as failed, rebuild it.
 164         git_dir: Git directory containing source repository
 165         num_jobs: Number of jobs to run at once (passed to make as -j)
 166         num_threads: Number of builder threads to run
 167         out_queue: Queue of results to process
 168         re_make_err: Compiled regular expression for ignore_lines
 169         queue: Queue of jobs to run
 170         threads: List of active threads
 171         toolchains: Toolchains object to use for building
 172         upto: Current commit number we are building (0.count-1)
 173         warned: Number of builds that produced at least one warning
 174         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 175             incremental building, where buildman reconfigures on the first
 176             commit for a baord, and then just does an incremental build for
 177             the following commits. In fact buildman will reconfigure and
 178             retry for any failing commits, so generally the only effect of
 179             this option is to slow things down.
 180         in_tree: Build U-Boot in-tree instead of specifying an output
 181             directory separate from the source code. This option is really
 182             only useful for testing in-tree builds.
 183         work_in_output: Use the output directory as the work directory and
 184             don't write to a separate output directory.
 185
 186     Private members:
 187         _base_board_dict: Last-summarised Dict of boards
 188         _base_err_lines: Last-summarised list of errors
 189         _base_warn_lines: Last-summarised list of warnings
 190         _build_period_us: Time taken for a single build (float object).
 191         _complete_delay: Expected delay until completion (timedelta)
 192         _next_delay_update: Next time we plan to display a progress update
 193                 (datatime)
 194         _show_unknown: Show unknown boards (those not built) in summary
 195         _start_time: Start time for the build
 196         _timestamps: List of timestamps for the completion of the last
 197             last _timestamp_count builds. Each is a datetime object.
 198         _timestamp_count: Number of timestamps to keep in our list.
 199         _working_dir: Base working directory containing all threads
 200     """
 201     class Outcome:
 202         """Records a build outcome for a single make invocation
 203
 204         Public Members:
 205             rc: Outcome value (OUTCOME_...)
 206             err_lines: List of error lines or [] if none
 207             sizes: Dictionary of image size information, keyed by filename
 208                 - Each value is itself a dictionary containing
 209                     values for 'text', 'data' and 'bss', being the integer
 210                     size in bytes of each section.
 211             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 212                     value is itself a dictionary:
 213                         key: function name
 214                         value: Size of function in bytes
 215             config: Dictionary keyed by filename - e.g. '.config'. Each
 216                     value is itself a dictionary:
 217                         key: config name
 218                         value: config value
 219             environment: Dictionary keyed by environment variable, Each
 220                      value is the value of environment variable.
 221         """
 222         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 223                      environment):
 224             self.rc = rc
 225             self.err_lines = err_lines
 226             self.sizes = sizes
 227             self.func_sizes = func_sizes
 228             self.config = config
 229             self.environment = environment
 230
 231     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 232                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 233                  no_subdirs=False, full_path=False, verbose_build=False,
 234                  mrproper=False, per_board_out_dir=False,
 235                  config_only=False, squash_config_y=False,
 236                  warnings_as_errors=False, work_in_output=False):
 237         """Create a new Builder object
 238
 239         Args:
 240             toolchains: Toolchains object to use for building
 241             base_dir: Base directory to use for builder
 242             git_dir: Git directory containing source repository
 243             num_threads: Number of builder threads to run
 244             num_jobs: Number of jobs to run at once (passed to make as -j)
 245             gnu_make: the command name of GNU Make.
 246             checkout: True to check out source, False to skip that step.
 247                 This is used for testing.
 248             show_unknown: Show unknown boards (those not built) in summary
 249             step: 1 to process every commit, n to process every nth commit
 250             no_subdirs: Don't create subdirectories when building current
 251                 source for a single board
 252             full_path: Return the full path in CROSS_COMPILE and don't set
 253                 PATH
 254             verbose_build: Run build with V=1 and don't use 'make -s'
 255             mrproper: Always run 'make mrproper' when configuring
 256             per_board_out_dir: Build in a separate persistent directory per
 257                 board rather than a thread-specific directory
 258             config_only: Only configure each build, don't build it
 259             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 260             warnings_as_errors: Treat all compiler warnings as errors
 261             work_in_output: Use the output directory as the work directory and
 262                 don't write to a separate output directory.
 263         """
 264         self.toolchains = toolchains
 265         self.base_dir = base_dir
 266         if work_in_output:
 267             self._working_dir = base_dir
 268         else:
 269             self._working_dir = os.path.join(base_dir, '.bm-work')
 270         self.threads = []
 271         self.do_make = self.Make
 272         self.gnu_make = gnu_make
 273         self.checkout = checkout
 274         self.num_threads = num_threads
 275         self.num_jobs = num_jobs
 276         self.already_done = 0
 277         self.force_build = False
 278         self.git_dir = git_dir
 279         self._show_unknown = show_unknown
 280         self._timestamp_count = 10
 281         self._build_period_us = None
 282         self._complete_delay = None
 283         self._next_delay_update = datetime.now()
 284         self._start_time = datetime.now()
 285         self.force_config_on_failure = True
 286         self.force_build_failures = False
 287         self.force_reconfig = False
 288         self._step = step
 289         self.in_tree = False
 290         self._error_lines = 0
 291         self.no_subdirs = no_subdirs
 292         self.full_path = full_path
 293         self.verbose_build = verbose_build
 294         self.config_only = config_only
 295         self.squash_config_y = squash_config_y
 296         self.config_filenames = BASE_CONFIG_FILENAMES
 297         self.work_in_output = work_in_output
 298         if not self.squash_config_y:
 299             self.config_filenames += EXTRA_CONFIG_FILENAMES
 300
 301         self.warnings_as_errors = warnings_as_errors
 302         self.col = terminal.Color()
 303
 304         self._re_function = re.compile('(.*): In function.*')
 305         self._re_files = re.compile('In file included from.*')
 306         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 307         self._re_dtb_warning = re.compile('(.*): Warning .*')
 308         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 309         self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
 310                                                 re.MULTILINE | re.DOTALL)
 311
 312         self.queue = queue.Queue()
 313         self.out_queue = queue.Queue()
 314         for i in range(self.num_threads):
 315             t = builderthread.BuilderThread(self, i, mrproper,
 316                     per_board_out_dir)
 317             t.setDaemon(True)
 318             t.start()
 319             self.threads.append(t)
 320
 321         t = builderthread.ResultThread(self)
 322         t.setDaemon(True)
 323         t.start()
 324         self.threads.append(t)
 325
 326         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 327         self.re_make_err = re.compile('|'.join(ignore_lines))
 328
 329         # Handle existing graceful with SIGINT / Ctrl-C
 330         signal.signal(signal.SIGINT, self.signal_handler)
 331
 332     def __del__(self):
 333         """Get rid of all threads created by the builder"""
 334         for t in self.threads:
 335             del t
 336
 337     def signal_handler(self, signal, frame):
 338         sys.exit(1)
 339
 340     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 341                           show_detail=False, show_bloat=False,
 342                           list_error_boards=False, show_config=False,
 343                           show_environment=False, filter_dtb_warnings=False,
 344                           filter_migration_warnings=False):
 345         """Setup display options for the builder.
 346
 347         Args:
 348             show_errors: True to show summarised error/warning info
 349             show_sizes: Show size deltas
 350             show_detail: Show size delta detail for each board if show_sizes
 351             show_bloat: Show detail for each function
 352             list_error_boards: Show the boards which caused each error/warning
 353             show_config: Show config deltas
 354             show_environment: Show environment deltas
 355             filter_dtb_warnings: Filter out any warnings from the device-tree
 356                 compiler
 357             filter_migration_warnings: Filter out any warnings about migrating
 358                 a board to driver model
 359         """
 360         self._show_errors = show_errors
 361         self._show_sizes = show_sizes
 362         self._show_detail = show_detail
 363         self._show_bloat = show_bloat
 364         self._list_error_boards = list_error_boards
 365         self._show_config = show_config
 366         self._show_environment = show_environment
 367         self._filter_dtb_warnings = filter_dtb_warnings
 368         self._filter_migration_warnings = filter_migration_warnings
 369
 370     def _AddTimestamp(self):
 371         """Add a new timestamp to the list and record the build period.
 372
 373         The build period is the length of time taken to perform a single
 374         build (one board, one commit).
 375         """
 376         now = datetime.now()
 377         self._timestamps.append(now)
 378         count = len(self._timestamps)
 379         delta = self._timestamps[-1] - self._timestamps[0]
 380         seconds = delta.total_seconds()
 381
 382         # If we have enough data, estimate build period (time taken for a
 383         # single build) and therefore completion time.
 384         if count > 1 and self._next_delay_update < now:
 385             self._next_delay_update = now + timedelta(seconds=2)
 386             if seconds > 0:
 387                 self._build_period = float(seconds) / count
 388                 todo = self.count - self.upto
 389                 self._complete_delay = timedelta(microseconds=
 390                         self._build_period * todo * 1000000)
 391                 # Round it
 392                 self._complete_delay -= timedelta(
 393                         microseconds=self._complete_delay.microseconds)
 394
 395         if seconds > 60:
 396             self._timestamps.popleft()
 397             count -= 1
 398
 399     def SelectCommit(self, commit, checkout=True):
 400         """Checkout the selected commit for this build
 401         """
 402         self.commit = commit
 403         if checkout and self.checkout:
 404             gitutil.Checkout(commit.hash)
 405
 406     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 407         """Run make
 408
 409         Args:
 410             commit: Commit object that is being built
 411             brd: Board object that is being built
 412             stage: Stage that we are at (mrproper, config, build)
 413             cwd: Directory where make should be run
 414             args: Arguments to pass to make
 415             kwargs: Arguments to pass to command.RunPipe()
 416         """
 417         cmd = [self.gnu_make] + list(args)
 418         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 419                 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
 420         if self.verbose_build:
 421             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 422             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 423         return result
 424
 425     def ProcessResult(self, result):
 426         """Process the result of a build, showing progress information
 427
 428         Args:
 429             result: A CommandResult object, which indicates the result for
 430                     a single build
 431         """
 432         col = terminal.Color()
 433         if result:
 434             target = result.brd.target
 435
 436             self.upto += 1
 437             if result.return_code != 0:
 438                 self.fail += 1
 439             elif result.stderr:
 440                 self.warned += 1
 441             if result.already_done:
 442                 self.already_done += 1
 443             if self._verbose:
 444                 terminal.PrintClear()
 445                 boards_selected = {target : result.brd}
 446                 self.ResetResultSummary(boards_selected)
 447                 self.ProduceResultSummary(result.commit_upto, self.commits,
 448                                           boards_selected)
 449         else:
 450             target = '(starting)'
 451
 452         # Display separate counts for ok, warned and fail
 453         ok = self.upto - self.warned - self.fail
 454         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 455         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 456         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 457
 458         line += ' /%-5d  ' % self.count
 459         remaining = self.count - self.upto
 460         if remaining:
 461             line += self.col.Color(self.col.MAGENTA, ' -%-5d  ' % remaining)
 462         else:
 463             line += ' ' * 8
 464
 465         # Add our current completion time estimate
 466         self._AddTimestamp()
 467         if self._complete_delay:
 468             line += '%s  : ' % self._complete_delay
 469
 470         line += target
 471         terminal.PrintClear()
 472         Print(line, newline=False, limit_to_line=True)
 473
 474     def _GetOutputDir(self, commit_upto):
 475         """Get the name of the output directory for a commit number
 476
 477         The output directory is typically .../<branch>/<commit>.
 478
 479         Args:
 480             commit_upto: Commit number to use (0..self.count-1)
 481         """
 482         commit_dir = None
 483         if self.commits:
 484             commit = self.commits[commit_upto]
 485             subject = commit.subject.translate(trans_valid_chars)
 486             # See _GetOutputSpaceRemovals() which parses this name
 487             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
 488                     self.commit_count, commit.hash, subject[:20]))
 489         elif not self.no_subdirs:
 490             commit_dir = 'current'
 491         if not commit_dir:
 492             return self.base_dir
 493         return os.path.join(self.base_dir, commit_dir)
 494
 495     def GetBuildDir(self, commit_upto, target):
 496         """Get the name of the build directory for a commit number
 497
 498         The build directory is typically .../<branch>/<commit>/<target>.
 499
 500         Args:
 501             commit_upto: Commit number to use (0..self.count-1)
 502             target: Target name
 503         """
 504         output_dir = self._GetOutputDir(commit_upto)
 505         return os.path.join(output_dir, target)
 506
 507     def GetDoneFile(self, commit_upto, target):
 508         """Get the name of the done file for a commit number
 509
 510         Args:
 511             commit_upto: Commit number to use (0..self.count-1)
 512             target: Target name
 513         """
 514         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 515
 516     def GetSizesFile(self, commit_upto, target):
 517         """Get the name of the sizes file for a commit number
 518
 519         Args:
 520             commit_upto: Commit number to use (0..self.count-1)
 521             target: Target name
 522         """
 523         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 524
 525     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 526         """Get the name of the funcsizes file for a commit number and ELF file
 527
 528         Args:
 529             commit_upto: Commit number to use (0..self.count-1)
 530             target: Target name
 531             elf_fname: Filename of elf image
 532         """
 533         return os.path.join(self.GetBuildDir(commit_upto, target),
 534                             '%s.sizes' % elf_fname.replace('/', '-'))
 535
 536     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 537         """Get the name of the objdump file for a commit number and ELF file
 538
 539         Args:
 540             commit_upto: Commit number to use (0..self.count-1)
 541             target: Target name
 542             elf_fname: Filename of elf image
 543         """
 544         return os.path.join(self.GetBuildDir(commit_upto, target),
 545                             '%s.objdump' % elf_fname.replace('/', '-'))
 546
 547     def GetErrFile(self, commit_upto, target):
 548         """Get the name of the err file for a commit number
 549
 550         Args:
 551             commit_upto: Commit number to use (0..self.count-1)
 552             target: Target name
 553         """
 554         output_dir = self.GetBuildDir(commit_upto, target)
 555         return os.path.join(output_dir, 'err')
 556
 557     def FilterErrors(self, lines):
 558         """Filter out errors in which we have no interest
 559
 560         We should probably use map().
 561
 562         Args:
 563             lines: List of error lines, each a string
 564         Returns:
 565             New list with only interesting lines included
 566         """
 567         out_lines = []
 568         if self._filter_migration_warnings:
 569             text = '\n'.join(lines)
 570             text = self._re_migration_warning.sub('', text)
 571             lines = text.splitlines()
 572         for line in lines:
 573             if self.re_make_err.search(line):
 574                 continue
 575             if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
 576                 continue
 577             out_lines.append(line)
 578         return out_lines
 579
 580     def ReadFuncSizes(self, fname, fd):
 581         """Read function sizes from the output of 'nm'
 582
 583         Args:
 584             fd: File containing data to read
 585             fname: Filename we are reading from (just for errors)
 586
 587         Returns:
 588             Dictionary containing size of each function in bytes, indexed by
 589             function name.
 590         """
 591         sym = {}
 592         for line in fd.readlines():
 593             try:
 594                 if line.strip():
 595                     size, type, name = line[:-1].split()
 596             except:
 597                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 598                 continue
 599             if type in 'tTdDbB':
 600                 # function names begin with '.' on 64-bit powerpc
 601                 if '.' in name[1:]:
 602                     name = 'static.' + name.split('.')[0]
 603                 sym[name] = sym.get(name, 0) + int(size, 16)
 604         return sym
 605
 606     def _ProcessConfig(self, fname):
 607         """Read in a .config, autoconf.mk or autoconf.h file
 608
 609         This function handles all config file types. It ignores comments and
 610         any #defines which don't start with CONFIG_.
 611
 612         Args:
 613             fname: Filename to read
 614
 615         Returns:
 616             Dictionary:
 617                 key: Config name (e.g. CONFIG_DM)
 618                 value: Config value (e.g. 1)
 619         """
 620         config = {}
 621         if os.path.exists(fname):
 622             with open(fname) as fd:
 623                 for line in fd:
 624                     line = line.strip()
 625                     if line.startswith('#define'):
 626                         values = line[8:].split(' ', 1)
 627                         if len(values) > 1:
 628                             key, value = values
 629                         else:
 630                             key = values[0]
 631                             value = '1' if self.squash_config_y else ''
 632                         if not key.startswith('CONFIG_'):
 633                             continue
 634                     elif not line or line[0] in ['#', '*', '/']:
 635                         continue
 636                     else:
 637                         key, value = line.split('=', 1)
 638                     if self.squash_config_y and value == 'y':
 639                         value = '1'
 640                     config[key] = value
 641         return config
 642
 643     def _ProcessEnvironment(self, fname):
 644         """Read in a uboot.env file
 645
 646         This function reads in environment variables from a file.
 647
 648         Args:
 649             fname: Filename to read
 650
 651         Returns:
 652             Dictionary:
 653                 key: environment variable (e.g. bootlimit)
 654                 value: value of environment variable (e.g. 1)
 655         """
 656         environment = {}
 657         if os.path.exists(fname):
 658             with open(fname) as fd:
 659                 for line in fd.read().split('\0'):
 660                     try:
 661                         key, value = line.split('=', 1)
 662                         environment[key] = value
 663                     except ValueError:
 664                         # ignore lines we can't parse
 665                         pass
 666         return environment
 667
 668     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 669                         read_config, read_environment):
 670         """Work out the outcome of a build.
 671
 672         Args:
 673             commit_upto: Commit number to check (0..n-1)
 674             target: Target board to check
 675             read_func_sizes: True to read function size information
 676             read_config: True to read .config and autoconf.h files
 677             read_environment: True to read uboot.env files
 678
 679         Returns:
 680             Outcome object
 681         """
 682         done_file = self.GetDoneFile(commit_upto, target)
 683         sizes_file = self.GetSizesFile(commit_upto, target)
 684         sizes = {}
 685         func_sizes = {}
 686         config = {}
 687         environment = {}
 688         if os.path.exists(done_file):
 689             with open(done_file, 'r') as fd:
 690                 try:
 691                     return_code = int(fd.readline())
 692                 except ValueError:
 693                     # The file may be empty due to running out of disk space.
 694                     # Try a rebuild
 695                     return_code = 1
 696                 err_lines = []
 697                 err_file = self.GetErrFile(commit_upto, target)
 698                 if os.path.exists(err_file):
 699                     with open(err_file, 'r') as fd:
 700                         err_lines = self.FilterErrors(fd.readlines())
 701
 702                 # Decide whether the build was ok, failed or created warnings
 703                 if return_code:
 704                     rc = OUTCOME_ERROR
 705                 elif len(err_lines):
 706                     rc = OUTCOME_WARNING
 707                 else:
 708                     rc = OUTCOME_OK
 709
 710                 # Convert size information to our simple format
 711                 if os.path.exists(sizes_file):
 712                     with open(sizes_file, 'r') as fd:
 713                         for line in fd.readlines():
 714                             values = line.split()
 715                             rodata = 0
 716                             if len(values) > 6:
 717                                 rodata = int(values[6], 16)
 718                             size_dict = {
 719                                 'all' : int(values[0]) + int(values[1]) +
 720                                         int(values[2]),
 721                                 'text' : int(values[0]) - rodata,
 722                                 'data' : int(values[1]),
 723                                 'bss' : int(values[2]),
 724                                 'rodata' : rodata,
 725                             }
 726                             sizes[values[5]] = size_dict
 727
 728             if read_func_sizes:
 729                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 730                 for fname in glob.glob(pattern):
 731                     with open(fname, 'r') as fd:
 732                         dict_name = os.path.basename(fname).replace('.sizes',
 733                                                                     '')
 734                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 735
 736             if read_config:
 737                 output_dir = self.GetBuildDir(commit_upto, target)
 738                 for name in self.config_filenames:
 739                     fname = os.path.join(output_dir, name)
 740                     config[name] = self._ProcessConfig(fname)
 741
 742             if read_environment:
 743                 output_dir = self.GetBuildDir(commit_upto, target)
 744                 fname = os.path.join(output_dir, 'uboot.env')
 745                 environment = self._ProcessEnvironment(fname)
 746
 747             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 748                                    environment)
 749
 750         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 751
 752     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 753                          read_config, read_environment):
 754         """Calculate a summary of the results of building a commit.
 755
 756         Args:
 757             board_selected: Dict containing boards to summarise
 758             commit_upto: Commit number to summarize (0..self.count-1)
 759             read_func_sizes: True to read function size information
 760             read_config: True to read .config and autoconf.h files
 761             read_environment: True to read uboot.env files
 762
 763         Returns:
 764             Tuple:
 765                 Dict containing boards which passed building this commit.
 766                     keyed by board.target
 767                 List containing a summary of error lines
 768                 Dict keyed by error line, containing a list of the Board
 769                     objects with that error
 770                 List containing a summary of warning lines
 771                 Dict keyed by error line, containing a list of the Board
 772                     objects with that warning
 773                 Dictionary keyed by board.target. Each value is a dictionary:
 774                     key: filename - e.g. '.config'
 775                     value is itself a dictionary:
 776                         key: config name
 777                         value: config value
 778                 Dictionary keyed by board.target. Each value is a dictionary:
 779                     key: environment variable
 780                     value: value of environment variable
 781         """
 782         def AddLine(lines_summary, lines_boards, line, board):
 783             line = line.rstrip()
 784             if line in lines_boards:
 785                 lines_boards[line].append(board)
 786             else:
 787                 lines_boards[line] = [board]
 788                 lines_summary.append(line)
 789
 790         board_dict = {}
 791         err_lines_summary = []
 792         err_lines_boards = {}
 793         warn_lines_summary = []
 794         warn_lines_boards = {}
 795         config = {}
 796         environment = {}
 797
 798         for board in boards_selected.values():
 799             outcome = self.GetBuildOutcome(commit_upto, board.target,
 800                                            read_func_sizes, read_config,
 801                                            read_environment)
 802             board_dict[board.target] = outcome
 803             last_func = None
 804             last_was_warning = False
 805             for line in outcome.err_lines:
 806                 if line:
 807                     if (self._re_function.match(line) or
 808                             self._re_files.match(line)):
 809                         last_func = line
 810                     else:
 811                         is_warning = (self._re_warning.match(line) or
 812                                       self._re_dtb_warning.match(line))
 813                         is_note = self._re_note.match(line)
 814                         if is_warning or (last_was_warning and is_note):
 815                             if last_func:
 816                                 AddLine(warn_lines_summary, warn_lines_boards,
 817                                         last_func, board)
 818                             AddLine(warn_lines_summary, warn_lines_boards,
 819                                     line, board)
 820                         else:
 821                             if last_func:
 822                                 AddLine(err_lines_summary, err_lines_boards,
 823                                         last_func, board)
 824                             AddLine(err_lines_summary, err_lines_boards,
 825                                     line, board)
 826                         last_was_warning = is_warning
 827                         last_func = None
 828             tconfig = Config(self.config_filenames, board.target)
 829             for fname in self.config_filenames:
 830                 if outcome.config:
 831                     for key, value in outcome.config[fname].items():
 832                         tconfig.Add(fname, key, value)
 833             config[board.target] = tconfig
 834
 835             tenvironment = Environment(board.target)
 836             if outcome.environment:
 837                 for key, value in outcome.environment.items():
 838                     tenvironment.Add(key, value)
 839             environment[board.target] = tenvironment
 840
 841         return (board_dict, err_lines_summary, err_lines_boards,
 842                 warn_lines_summary, warn_lines_boards, config, environment)
 843
 844     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 845         """Add an output to our list of outcomes for each architecture
 846
 847         This simple function adds failing boards (changes) to the
 848         relevant architecture string, so we can print the results out
 849         sorted by architecture.
 850
 851         Args:
 852              board_dict: Dict containing all boards
 853              arch_list: Dict keyed by arch name. Value is a string containing
 854                     a list of board names which failed for that arch.
 855              changes: List of boards to add to arch_list
 856              color: terminal.Colour object
 857         """
 858         done_arch = {}
 859         for target in changes:
 860             if target in board_dict:
 861                 arch = board_dict[target].arch
 862             else:
 863                 arch = 'unknown'
 864             str = self.col.Color(color, ' ' + target)
 865             if not arch in done_arch:
 866                 str = ' %s  %s' % (self.col.Color(color, char), str)
 867                 done_arch[arch] = True
 868             if not arch in arch_list:
 869                 arch_list[arch] = str
 870             else:
 871                 arch_list[arch] += str
 872
 873
 874     def ColourNum(self, num):
 875         color = self.col.RED if num > 0 else self.col.GREEN
 876         if num == 0:
 877             return '0'
 878         return self.col.Color(color, str(num))
 879
 880     def ResetResultSummary(self, board_selected):
 881         """Reset the results summary ready for use.
 882
 883         Set up the base board list to be all those selected, and set the
 884         error lines to empty.
 885
 886         Following this, calls to PrintResultSummary() will use this
 887         information to work out what has changed.
 888
 889         Args:
 890             board_selected: Dict containing boards to summarise, keyed by
 891                 board.target
 892         """
 893         self._base_board_dict = {}
 894         for board in board_selected:
 895             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 896                                                            {})
 897         self._base_err_lines = []
 898         self._base_warn_lines = []
 899         self._base_err_line_boards = {}
 900         self._base_warn_line_boards = {}
 901         self._base_config = None
 902         self._base_environment = None
 903
 904     def PrintFuncSizeDetail(self, fname, old, new):
 905         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 906         delta, common = [], {}
 907
 908         for a in old:
 909             if a in new:
 910                 common[a] = 1
 911
 912         for name in old:
 913             if name not in common:
 914                 remove += 1
 915                 down += old[name]
 916                 delta.append([-old[name], name])
 917
 918         for name in new:
 919             if name not in common:
 920                 add += 1
 921                 up += new[name]
 922                 delta.append([new[name], name])
 923
 924         for name in common:
 925                 diff = new.get(name, 0) - old.get(name, 0)
 926                 if diff > 0:
 927                     grow, up = grow + 1, up + diff
 928                 elif diff < 0:
 929                     shrink, down = shrink + 1, down - diff
 930                 delta.append([diff, name])
 931
 932         delta.sort()
 933         delta.reverse()
 934
 935         args = [add, -remove, grow, -shrink, up, -down, up - down]
 936         if max(args) == 0 and min(args) == 0:
 937             return
 938         args = [self.ColourNum(x) for x in args]
 939         indent = ' ' * 15
 940         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
 941               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
 942         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
 943                                          'delta'))
 944         for diff, name in delta:
 945             if diff:
 946                 color = self.col.RED if diff > 0 else self.col.GREEN
 947                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
 948                         old.get(name, '-'), new.get(name,'-'), diff)
 949                 Print(msg, colour=color)
 950
 951
 952     def PrintSizeDetail(self, target_list, show_bloat):
 953         """Show details size information for each board
 954
 955         Args:
 956             target_list: List of targets, each a dict containing:
 957                     'target': Target name
 958                     'total_diff': Total difference in bytes across all areas
 959                     <part_name>: Difference for that part
 960             show_bloat: Show detail for each function
 961         """
 962         targets_by_diff = sorted(target_list, reverse=True,
 963         key=lambda x: x['_total_diff'])
 964         for result in targets_by_diff:
 965             printed_target = False
 966             for name in sorted(result):
 967                 diff = result[name]
 968                 if name.startswith('_'):
 969                     continue
 970                 if diff != 0:
 971                     color = self.col.RED if diff > 0 else self.col.GREEN
 972                 msg = ' %s %+d' % (name, diff)
 973                 if not printed_target:
 974                     Print('%10s  %-15s:' % ('', result['_target']),
 975                           newline=False)
 976                     printed_target = True
 977                 Print(msg, colour=color, newline=False)
 978             if printed_target:
 979                 Print()
 980                 if show_bloat:
 981                     target = result['_target']
 982                     outcome = result['_outcome']
 983                     base_outcome = self._base_board_dict[target]
 984                     for fname in outcome.func_sizes:
 985                         self.PrintFuncSizeDetail(fname,
 986                                                  base_outcome.func_sizes[fname],
 987                                                  outcome.func_sizes[fname])
 988
 989
 990     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
 991                          show_bloat):
 992         """Print a summary of image sizes broken down by section.
 993
 994         The summary takes the form of one line per architecture. The
 995         line contains deltas for each of the sections (+ means the section
 996         got bigger, - means smaller). The numbers are the average number
 997         of bytes that a board in this section increased by.
 998
 999         For example:
1000            powerpc: (622 boards)   text -0.0
1001           arm: (285 boards)   text -0.0
1002           nds32: (3 boards)   text -8.0
1003
1004         Args:
1005             board_selected: Dict containing boards to summarise, keyed by
1006                 board.target
1007             board_dict: Dict containing boards for which we built this
1008                 commit, keyed by board.target. The value is an Outcome object.
1009             show_detail: Show size delta detail for each board
1010             show_bloat: Show detail for each function
1011         """
1012         arch_list = {}
1013         arch_count = {}
1014
1015         # Calculate changes in size for different image parts
1016         # The previous sizes are in Board.sizes, for each board
1017         for target in board_dict:
1018             if target not in board_selected:
1019                 continue
1020             base_sizes = self._base_board_dict[target].sizes
1021             outcome = board_dict[target]
1022             sizes = outcome.sizes
1023
1024             # Loop through the list of images, creating a dict of size
1025             # changes for each image/part. We end up with something like
1026             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1027             # which means that U-Boot data increased by 5 bytes and SPL
1028             # text decreased by 4.
1029             err = {'_target' : target}
1030             for image in sizes:
1031                 if image in base_sizes:
1032                     base_image = base_sizes[image]
1033                     # Loop through the text, data, bss parts
1034                     for part in sorted(sizes[image]):
1035                         diff = sizes[image][part] - base_image[part]
1036                         col = None
1037                         if diff:
1038                             if image == 'u-boot':
1039                                 name = part
1040                             else:
1041                                 name = image + ':' + part
1042                             err[name] = diff
1043             arch = board_selected[target].arch
1044             if not arch in arch_count:
1045                 arch_count[arch] = 1
1046             else:
1047                 arch_count[arch] += 1
1048             if not sizes:
1049                 pass    # Only add to our list when we have some stats
1050             elif not arch in arch_list:
1051                 arch_list[arch] = [err]
1052             else:
1053                 arch_list[arch].append(err)
1054
1055         # We now have a list of image size changes sorted by arch
1056         # Print out a summary of these
1057         for arch, target_list in arch_list.items():
1058             # Get total difference for each type
1059             totals = {}
1060             for result in target_list:
1061                 total = 0
1062                 for name, diff in result.items():
1063                     if name.startswith('_'):
1064                         continue
1065                     total += diff
1066                     if name in totals:
1067                         totals[name] += diff
1068                     else:
1069                         totals[name] = diff
1070                 result['_total_diff'] = total
1071                 result['_outcome'] = board_dict[result['_target']]
1072
1073             count = len(target_list)
1074             printed_arch = False
1075             for name in sorted(totals):
1076                 diff = totals[name]
1077                 if diff:
1078                     # Display the average difference in this name for this
1079                     # architecture
1080                     avg_diff = float(diff) / count
1081                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1082                     msg = ' %s %+1.1f' % (name, avg_diff)
1083                     if not printed_arch:
1084                         Print('%10s: (for %d/%d boards)' % (arch, count,
1085                               arch_count[arch]), newline=False)
1086                         printed_arch = True
1087                     Print(msg, colour=color, newline=False)
1088
1089             if printed_arch:
1090                 Print()
1091                 if show_detail:
1092                     self.PrintSizeDetail(target_list, show_bloat)
1093
1094
1095     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1096                            err_line_boards, warn_lines, warn_line_boards,
1097                            config, environment, show_sizes, show_detail,
1098                            show_bloat, show_config, show_environment):
1099         """Compare results with the base results and display delta.
1100
1101         Only boards mentioned in board_selected will be considered. This
1102         function is intended to be called repeatedly with the results of
1103         each commit. It therefore shows a 'diff' between what it saw in
1104         the last call and what it sees now.
1105
1106         Args:
1107             board_selected: Dict containing boards to summarise, keyed by
1108                 board.target
1109             board_dict: Dict containing boards for which we built this
1110                 commit, keyed by board.target. The value is an Outcome object.
1111             err_lines: A list of errors for this commit, or [] if there is
1112                 none, or we don't want to print errors
1113             err_line_boards: Dict keyed by error line, containing a list of
1114                 the Board objects with that error
1115             warn_lines: A list of warnings for this commit, or [] if there is
1116                 none, or we don't want to print errors
1117             warn_line_boards: Dict keyed by warning line, containing a list of
1118                 the Board objects with that warning
1119             config: Dictionary keyed by filename - e.g. '.config'. Each
1120                     value is itself a dictionary:
1121                         key: config name
1122                         value: config value
1123             environment: Dictionary keyed by environment variable, Each
1124                      value is the value of environment variable.
1125             show_sizes: Show image size deltas
1126             show_detail: Show size delta detail for each board if show_sizes
1127             show_bloat: Show detail for each function
1128             show_config: Show config changes
1129             show_environment: Show environment changes
1130         """
1131         def _BoardList(line, line_boards):
1132             """Helper function to get a line of boards containing a line
1133
1134             Args:
1135                 line: Error line to search for
1136                 line_boards: boards to search, each a Board
1137             Return:
1138                 List of boards with that error line, or [] if the user has not
1139                     requested such a list
1140             """
1141             boards = []
1142             board_set = set()
1143             if self._list_error_boards:
1144                 for board in line_boards[line]:
1145                     if not board in board_set:
1146                         boards.append(board)
1147                         board_set.add(board)
1148             return boards
1149
1150         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1151                             char):
1152             """Calculate the required output based on changes in errors
1153
1154             Args:
1155                 base_lines: List of errors/warnings for previous commit
1156                 base_line_boards: Dict keyed by error line, containing a list
1157                     of the Board objects with that error in the previous commit
1158                 lines: List of errors/warning for this commit, each a str
1159                 line_boards: Dict keyed by error line, containing a list
1160                     of the Board objects with that error in this commit
1161                 char: Character representing error ('') or warning ('w'). The
1162                     broken ('+') or fixed ('-') characters are added in this
1163                     function
1164
1165             Returns:
1166                 Tuple
1167                     List of ErrLine objects for 'better' lines
1168                     List of ErrLine objects for 'worse' lines
1169             """
1170             better_lines = []
1171             worse_lines = []
1172             for line in lines:
1173                 if line not in base_lines:
1174                     errline = ErrLine(char + '+', _BoardList(line, line_boards),
1175                                       line)
1176                     worse_lines.append(errline)
1177             for line in base_lines:
1178                 if line not in lines:
1179                     errline = ErrLine(char + '-',
1180                                       _BoardList(line, base_line_boards), line)
1181                     better_lines.append(errline)
1182             return better_lines, worse_lines
1183
1184         def _CalcConfig(delta, name, config):
1185             """Calculate configuration changes
1186
1187             Args:
1188                 delta: Type of the delta, e.g. '+'
1189                 name: name of the file which changed (e.g. .config)
1190                 config: configuration change dictionary
1191                     key: config name
1192                     value: config value
1193             Returns:
1194                 String containing the configuration changes which can be
1195                     printed
1196             """
1197             out = ''
1198             for key in sorted(config.keys()):
1199                 out += '%s=%s ' % (key, config[key])
1200             return '%s %s: %s' % (delta, name, out)
1201
1202         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1203             """Add changes in configuration to a list
1204
1205             Args:
1206                 lines: list to add to
1207                 name: config file name
1208                 config_plus: configurations added, dictionary
1209                     key: config name
1210                     value: config value
1211                 config_minus: configurations removed, dictionary
1212                     key: config name
1213                     value: config value
1214                 config_change: configurations changed, dictionary
1215                     key: config name
1216                     value: config value
1217             """
1218             if config_plus:
1219                 lines.append(_CalcConfig('+', name, config_plus))
1220             if config_minus:
1221                 lines.append(_CalcConfig('-', name, config_minus))
1222             if config_change:
1223                 lines.append(_CalcConfig('c', name, config_change))
1224
1225         def _OutputConfigInfo(lines):
1226             for line in lines:
1227                 if not line:
1228                     continue
1229                 if line[0] == '+':
1230                     col = self.col.GREEN
1231                 elif line[0] == '-':
1232                     col = self.col.RED
1233                 elif line[0] == 'c':
1234                     col = self.col.YELLOW
1235                 Print('   ' + line, newline=True, colour=col)
1236
1237         def _OutputErrLines(err_lines, colour):
1238             """Output the line of error/warning lines, if not empty
1239
1240             Also increments self._error_lines if err_lines not empty
1241
1242             Args:
1243                 err_lines: List of ErrLine objects, each an error or warning
1244                     line, possibly including a list of boards with that
1245                     error/warning
1246                 colour: Colour to use for output
1247             """
1248             if err_lines:
1249                 out_list = []
1250                 for line in err_lines:
1251                     boards = ''
1252                     names = [board.target for board in line.boards]
1253                     board_str = ' '.join(names) if names else ''
1254                     if board_str:
1255                         out = self.col.Color(colour, line.char + '(')
1256                         out += self.col.Color(self.col.MAGENTA, board_str,
1257                                               bright=False)
1258                         out += self.col.Color(colour, ') %s' % line.errline)
1259                     else:
1260                         out = self.col.Color(colour, line.char + line.errline)
1261                     out_list.append(out)
1262                 Print('\n'.join(out_list))
1263                 self._error_lines += 1
1264
1265
1266         ok_boards = []      # List of boards fixed since last commit
1267         warn_boards = []    # List of boards with warnings since last commit
1268         err_boards = []     # List of new broken boards since last commit
1269         new_boards = []     # List of boards that didn't exist last time
1270         unknown_boards = [] # List of boards that were not built
1271
1272         for target in board_dict:
1273             if target not in board_selected:
1274                 continue
1275
1276             # If the board was built last time, add its outcome to a list
1277             if target in self._base_board_dict:
1278                 base_outcome = self._base_board_dict[target].rc
1279                 outcome = board_dict[target]
1280                 if outcome.rc == OUTCOME_UNKNOWN:
1281                     unknown_boards.append(target)
1282                 elif outcome.rc < base_outcome:
1283                     if outcome.rc == OUTCOME_WARNING:
1284                         warn_boards.append(target)
1285                     else:
1286                         ok_boards.append(target)
1287                 elif outcome.rc > base_outcome:
1288                     if outcome.rc == OUTCOME_WARNING:
1289                         warn_boards.append(target)
1290                     else:
1291                         err_boards.append(target)
1292             else:
1293                 new_boards.append(target)
1294
1295         # Get a list of errors and warnings that have appeared, and disappeared
1296         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1297                 self._base_err_line_boards, err_lines, err_line_boards, '')
1298         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1299                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1300
1301         # Display results by arch
1302         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1303                 worse_err, better_err, worse_warn, better_warn)):
1304             arch_list = {}
1305             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1306                     self.col.GREEN)
1307             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1308                     self.col.YELLOW)
1309             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1310                     self.col.RED)
1311             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1312             if self._show_unknown:
1313                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1314                         self.col.MAGENTA)
1315             for arch, target_list in arch_list.items():
1316                 Print('%10s: %s' % (arch, target_list))
1317                 self._error_lines += 1
1318             _OutputErrLines(better_err, colour=self.col.GREEN)
1319             _OutputErrLines(worse_err, colour=self.col.RED)
1320             _OutputErrLines(better_warn, colour=self.col.CYAN)
1321             _OutputErrLines(worse_warn, colour=self.col.YELLOW)
1322
1323         if show_sizes:
1324             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1325                                   show_bloat)
1326
1327         if show_environment and self._base_environment:
1328             lines = []
1329
1330             for target in board_dict:
1331                 if target not in board_selected:
1332                     continue
1333
1334                 tbase = self._base_environment[target]
1335                 tenvironment = environment[target]
1336                 environment_plus = {}
1337                 environment_minus = {}
1338                 environment_change = {}
1339                 base = tbase.environment
1340                 for key, value in tenvironment.environment.items():
1341                     if key not in base:
1342                         environment_plus[key] = value
1343                 for key, value in base.items():
1344                     if key not in tenvironment.environment:
1345                         environment_minus[key] = value
1346                 for key, value in base.items():
1347                     new_value = tenvironment.environment.get(key)
1348                     if new_value and value != new_value:
1349                         desc = '%s -> %s' % (value, new_value)
1350                         environment_change[key] = desc
1351
1352                 _AddConfig(lines, target, environment_plus, environment_minus,
1353                            environment_change)
1354
1355             _OutputConfigInfo(lines)
1356
1357         if show_config and self._base_config:
1358             summary = {}
1359             arch_config_plus = {}
1360             arch_config_minus = {}
1361             arch_config_change = {}
1362             arch_list = []
1363
1364             for target in board_dict:
1365                 if target not in board_selected:
1366                     continue
1367                 arch = board_selected[target].arch
1368                 if arch not in arch_list:
1369                     arch_list.append(arch)
1370
1371             for arch in arch_list:
1372                 arch_config_plus[arch] = {}
1373                 arch_config_minus[arch] = {}
1374                 arch_config_change[arch] = {}
1375                 for name in self.config_filenames:
1376                     arch_config_plus[arch][name] = {}
1377                     arch_config_minus[arch][name] = {}
1378                     arch_config_change[arch][name] = {}
1379
1380             for target in board_dict:
1381                 if target not in board_selected:
1382                     continue
1383
1384                 arch = board_selected[target].arch
1385
1386                 all_config_plus = {}
1387                 all_config_minus = {}
1388                 all_config_change = {}
1389                 tbase = self._base_config[target]
1390                 tconfig = config[target]
1391                 lines = []
1392                 for name in self.config_filenames:
1393                     if not tconfig.config[name]:
1394                         continue
1395                     config_plus = {}
1396                     config_minus = {}
1397                     config_change = {}
1398                     base = tbase.config[name]
1399                     for key, value in tconfig.config[name].items():
1400                         if key not in base:
1401                             config_plus[key] = value
1402                             all_config_plus[key] = value
1403                     for key, value in base.items():
1404                         if key not in tconfig.config[name]:
1405                             config_minus[key] = value
1406                             all_config_minus[key] = value
1407                     for key, value in base.items():
1408                         new_value = tconfig.config.get(key)
1409                         if new_value and value != new_value:
1410                             desc = '%s -> %s' % (value, new_value)
1411                             config_change[key] = desc
1412                             all_config_change[key] = desc
1413
1414                     arch_config_plus[arch][name].update(config_plus)
1415                     arch_config_minus[arch][name].update(config_minus)
1416                     arch_config_change[arch][name].update(config_change)
1417
1418                     _AddConfig(lines, name, config_plus, config_minus,
1419                                config_change)
1420                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1421                            all_config_change)
1422                 summary[target] = '\n'.join(lines)
1423
1424             lines_by_target = {}
1425             for target, lines in summary.items():
1426                 if lines in lines_by_target:
1427                     lines_by_target[lines].append(target)
1428                 else:
1429                     lines_by_target[lines] = [target]
1430
1431             for arch in arch_list:
1432                 lines = []
1433                 all_plus = {}
1434                 all_minus = {}
1435                 all_change = {}
1436                 for name in self.config_filenames:
1437                     all_plus.update(arch_config_plus[arch][name])
1438                     all_minus.update(arch_config_minus[arch][name])
1439                     all_change.update(arch_config_change[arch][name])
1440                     _AddConfig(lines, name, arch_config_plus[arch][name],
1441                                arch_config_minus[arch][name],
1442                                arch_config_change[arch][name])
1443                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1444                 #arch_summary[target] = '\n'.join(lines)
1445                 if lines:
1446                     Print('%s:' % arch)
1447                     _OutputConfigInfo(lines)
1448
1449             for lines, targets in lines_by_target.items():
1450                 if not lines:
1451                     continue
1452                 Print('%s :' % ' '.join(sorted(targets)))
1453                 _OutputConfigInfo(lines.split('\n'))
1454
1455
1456         # Save our updated information for the next call to this function
1457         self._base_board_dict = board_dict
1458         self._base_err_lines = err_lines
1459         self._base_warn_lines = warn_lines
1460         self._base_err_line_boards = err_line_boards
1461         self._base_warn_line_boards = warn_line_boards
1462         self._base_config = config
1463         self._base_environment = environment
1464
1465         # Get a list of boards that did not get built, if needed
1466         not_built = []
1467         for board in board_selected:
1468             if not board in board_dict:
1469                 not_built.append(board)
1470         if not_built:
1471             Print("Boards not built (%d): %s" % (len(not_built),
1472                   ', '.join(not_built)))
1473
1474     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1475             (board_dict, err_lines, err_line_boards, warn_lines,
1476              warn_line_boards, config, environment) = self.GetResultSummary(
1477                     board_selected, commit_upto,
1478                     read_func_sizes=self._show_bloat,
1479                     read_config=self._show_config,
1480                     read_environment=self._show_environment)
1481             if commits:
1482                 msg = '%02d: %s' % (commit_upto + 1,
1483                         commits[commit_upto].subject)
1484                 Print(msg, colour=self.col.BLUE)
1485             self.PrintResultSummary(board_selected, board_dict,
1486                     err_lines if self._show_errors else [], err_line_boards,
1487                     warn_lines if self._show_errors else [], warn_line_boards,
1488                     config, environment, self._show_sizes, self._show_detail,
1489                     self._show_bloat, self._show_config, self._show_environment)
1490
1491     def ShowSummary(self, commits, board_selected):
1492         """Show a build summary for U-Boot for a given board list.
1493
1494         Reset the result summary, then repeatedly call GetResultSummary on
1495         each commit's results, then display the differences we see.
1496
1497         Args:
1498             commit: Commit objects to summarise
1499             board_selected: Dict containing boards to summarise
1500         """
1501         self.commit_count = len(commits) if commits else 1
1502         self.commits = commits
1503         self.ResetResultSummary(board_selected)
1504         self._error_lines = 0
1505
1506         for commit_upto in range(0, self.commit_count, self._step):
1507             self.ProduceResultSummary(commit_upto, commits, board_selected)
1508         if not self._error_lines:
1509             Print('(no errors to report)', colour=self.col.GREEN)
1510
1511
1512     def SetupBuild(self, board_selected, commits):
1513         """Set up ready to start a build.
1514
1515         Args:
1516             board_selected: Selected boards to build
1517             commits: Selected commits to build
1518         """
1519         # First work out how many commits we will build
1520         count = (self.commit_count + self._step - 1) // self._step
1521         self.count = len(board_selected) * count
1522         self.upto = self.warned = self.fail = 0
1523         self._timestamps = collections.deque()
1524
1525     def GetThreadDir(self, thread_num):
1526         """Get the directory path to the working dir for a thread.
1527
1528         Args:
1529             thread_num: Number of thread to check.
1530         """
1531         if self.work_in_output:
1532             return self._working_dir
1533         return os.path.join(self._working_dir, '%02d' % thread_num)
1534
1535     def _PrepareThread(self, thread_num, setup_git):
1536         """Prepare the working directory for a thread.
1537
1538         This clones or fetches the repo into the thread's work directory.
1539
1540         Args:
1541             thread_num: Thread number (0, 1, ...)
1542             setup_git: True to set up a git repo clone
1543         """
1544         thread_dir = self.GetThreadDir(thread_num)
1545         builderthread.Mkdir(thread_dir)
1546         git_dir = os.path.join(thread_dir, '.git')
1547
1548         # Clone the repo if it doesn't already exist
1549         # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1550         # we have a private index but uses the origin repo's contents?
1551         if setup_git and self.git_dir:
1552             src_dir = os.path.abspath(self.git_dir)
1553             if os.path.exists(git_dir):
1554                 Print('\rFetching repo for thread %d' % thread_num,
1555                       newline=False)
1556                 gitutil.Fetch(git_dir, thread_dir)
1557                 terminal.PrintClear()
1558             else:
1559                 Print('\rCloning repo for thread %d' % thread_num,
1560                       newline=False)
1561                 gitutil.Clone(src_dir, thread_dir)
1562                 terminal.PrintClear()
1563
1564     def _PrepareWorkingSpace(self, max_threads, setup_git):
1565         """Prepare the working directory for use.
1566
1567         Set up the git repo for each thread.
1568
1569         Args:
1570             max_threads: Maximum number of threads we expect to need.
1571             setup_git: True to set up a git repo clone
1572         """
1573         builderthread.Mkdir(self._working_dir)
1574         for thread in range(max_threads):
1575             self._PrepareThread(thread, setup_git)
1576
1577     def _GetOutputSpaceRemovals(self):
1578         """Get the output directories ready to receive files.
1579
1580         Figure out what needs to be deleted in the output directory before it
1581         can be used. We only delete old buildman directories which have the
1582         expected name pattern. See _GetOutputDir().
1583
1584         Returns:
1585             List of full paths of directories to remove
1586         """
1587         if not self.commits:
1588             return
1589         dir_list = []
1590         for commit_upto in range(self.commit_count):
1591             dir_list.append(self._GetOutputDir(commit_upto))
1592
1593         to_remove = []
1594         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1595             if dirname not in dir_list:
1596                 leaf = dirname[len(self.base_dir) + 1:]
1597                 m =  re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1598                 if m:
1599                     to_remove.append(dirname)
1600         return to_remove
1601
1602     def _PrepareOutputSpace(self):
1603         """Get the output directories ready to receive files.
1604
1605         We delete any output directories which look like ones we need to
1606         create. Having left over directories is confusing when the user wants
1607         to check the output manually.
1608         """
1609         to_remove = self._GetOutputSpaceRemovals()
1610         if to_remove:
1611             Print('Removing %d old build directories...' % len(to_remove),
1612                   newline=False)
1613             for dirname in to_remove:
1614                 shutil.rmtree(dirname)
1615             terminal.PrintClear()
1616
1617     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1618         """Build all commits for a list of boards
1619
1620         Args:
1621             commits: List of commits to be build, each a Commit object
1622             boards_selected: Dict of selected boards, key is target name,
1623                     value is Board object
1624             keep_outputs: True to save build output files
1625             verbose: Display build results as they are completed
1626         Returns:
1627             Tuple containing:
1628                 - number of boards that failed to build
1629                 - number of boards that issued warnings
1630         """
1631         self.commit_count = len(commits) if commits else 1
1632         self.commits = commits
1633         self._verbose = verbose
1634
1635         self.ResetResultSummary(board_selected)
1636         builderthread.Mkdir(self.base_dir, parents = True)
1637         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1638                 commits is not None)
1639         self._PrepareOutputSpace()
1640         Print('\rStarting build...', newline=False)
1641         self.SetupBuild(board_selected, commits)
1642         self.ProcessResult(None)
1643
1644         # Create jobs to build all commits for each board
1645         for brd in board_selected.values():
1646             job = builderthread.BuilderJob()
1647             job.board = brd
1648             job.commits = commits
1649             job.keep_outputs = keep_outputs
1650             job.work_in_output = self.work_in_output
1651             job.step = self._step
1652             self.queue.put(job)
1653
1654         term = threading.Thread(target=self.queue.join)
1655         term.setDaemon(True)
1656         term.start()
1657         while term.isAlive():
1658             term.join(100)
1659
1660         # Wait until we have processed all output
1661         self.out_queue.join()
1662         Print()
1663
1664         msg = 'Completed: %d total built' % self.count
1665         if self.already_done:
1666            msg += ' (%d previously' % self.already_done
1667            if self.already_done != self.count:
1668                msg += ', %d newly' % (self.count - self.already_done)
1669            msg += ')'
1670         duration = datetime.now() - self._start_time
1671         if duration > timedelta(microseconds=1000000):
1672             if duration.microseconds >= 500000:
1673                 duration = duration + timedelta(seconds=1)
1674             duration = duration - timedelta(microseconds=duration.microseconds)
1675             msg += ', duration %s' % duration
1676         Print(msg)
1677
1678         return (self.fail, self.warned)