tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 import builderthread
  21 import command
  22 import gitutil
  23 import terminal
  24 from terminal import Print
  25 import toolchain
  26
  27
  28 """
  29 Theory of Operation
  30
  31 Please see README for user documentation, and you should be familiar with
  32 that before trying to make sense of this.
  33
  34 Buildman works by keeping the machine as busy as possible, building different
  35 commits for different boards on multiple CPUs at once.
  36
  37 The source repo (self.git_dir) contains all the commits to be built. Each
  38 thread works on a single board at a time. It checks out the first commit,
  39 configures it for that board, then builds it. Then it checks out the next
  40 commit and builds it (typically without re-configuring). When it runs out
  41 of commits, it gets another job from the builder and starts again with that
  42 board.
  43
  44 Clearly the builder threads could work either way - they could check out a
  45 commit and then built it for all boards. Using separate directories for each
  46 commit/board pair they could leave their build product around afterwards
  47 also.
  48
  49 The intent behind building a single board for multiple commits, is to make
  50 use of incremental builds. Since each commit is built incrementally from
  51 the previous one, builds are faster. Reconfiguring for a different board
  52 removes all intermediate object files.
  53
  54 Many threads can be working at once, but each has its own working directory.
  55 When a thread finishes a build, it puts the output files into a result
  56 directory.
  57
  58 The base directory used by buildman is normally '../<branch>', i.e.
  59 a directory higher than the source repository and named after the branch
  60 being built.
  61
  62 Within the base directory, we have one subdirectory for each commit. Within
  63 that is one subdirectory for each board. Within that is the build output for
  64 that commit/board combination.
  65
  66 Buildman also create working directories for each thread, in a .bm-work/
  67 subdirectory in the base dir.
  68
  69 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  70 'seaboard', and say that us-net has two commits. We will have directories
  71 like this:
  72
  73 us-net/             base directory
  74     01_of_02_g4ed4ebc_net--Add-tftp-speed-/
  75         sandbox/
  76             u-boot.bin
  77         seaboard/
  78             u-boot.bin
  79     02_of_02_g4ed4ebc_net--Check-tftp-comp/
  80         sandbox/
  81             u-boot.bin
  82         seaboard/
  83             u-boot.bin
  84     .bm-work/
  85         00/         working directory for thread 0 (contains source checkout)
  86             build/  build output
  87         01/         working directory for thread 1
  88             build/  build output
  89         ...
  90 u-boot/             source directory
  91     .git/           repository
  92 """
  93
  94 # Possible build outcomes
  95 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
  96
  97 # Translate a commit subject into a valid filename (and handle unicode)
  98 trans_valid_chars = str.maketrans('/: ', '---')
  99
 100 BASE_CONFIG_FILENAMES = [
 101     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 102 ]
 103
 104 EXTRA_CONFIG_FILENAMES = [
 105     '.config', '.config-spl', '.config-tpl',
 106     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 107     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 108 ]
 109
 110 class Config:
 111     """Holds information about configuration settings for a board."""
 112     def __init__(self, config_filename, target):
 113         self.target = target
 114         self.config = {}
 115         for fname in config_filename:
 116             self.config[fname] = {}
 117
 118     def Add(self, fname, key, value):
 119         self.config[fname][key] = value
 120
 121     def __hash__(self):
 122         val = 0
 123         for fname in self.config:
 124             for key, value in self.config[fname].items():
 125                 print(key, value)
 126                 val = val ^ hash(key) & hash(value)
 127         return val
 128
 129 class Environment:
 130     """Holds information about environment variables for a board."""
 131     def __init__(self, target):
 132         self.target = target
 133         self.environment = {}
 134
 135     def Add(self, key, value):
 136         self.environment[key] = value
 137
 138 class Builder:
 139     """Class for building U-Boot for a particular commit.
 140
 141     Public members: (many should ->private)
 142         already_done: Number of builds already completed
 143         base_dir: Base directory to use for builder
 144         checkout: True to check out source, False to skip that step.
 145             This is used for testing.
 146         col: terminal.Color() object
 147         count: Number of commits to build
 148         do_make: Method to call to invoke Make
 149         fail: Number of builds that failed due to error
 150         force_build: Force building even if a build already exists
 151         force_config_on_failure: If a commit fails for a board, disable
 152             incremental building for the next commit we build for that
 153             board, so that we will see all warnings/errors again.
 154         force_build_failures: If a previously-built build (i.e. built on
 155             a previous run of buildman) is marked as failed, rebuild it.
 156         git_dir: Git directory containing source repository
 157         last_line_len: Length of the last line we printed (used for erasing
 158             it with new progress information)
 159         num_jobs: Number of jobs to run at once (passed to make as -j)
 160         num_threads: Number of builder threads to run
 161         out_queue: Queue of results to process
 162         re_make_err: Compiled regular expression for ignore_lines
 163         queue: Queue of jobs to run
 164         threads: List of active threads
 165         toolchains: Toolchains object to use for building
 166         upto: Current commit number we are building (0.count-1)
 167         warned: Number of builds that produced at least one warning
 168         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 169             incremental building, where buildman reconfigures on the first
 170             commit for a baord, and then just does an incremental build for
 171             the following commits. In fact buildman will reconfigure and
 172             retry for any failing commits, so generally the only effect of
 173             this option is to slow things down.
 174         in_tree: Build U-Boot in-tree instead of specifying an output
 175             directory separate from the source code. This option is really
 176             only useful for testing in-tree builds.
 177
 178     Private members:
 179         _base_board_dict: Last-summarised Dict of boards
 180         _base_err_lines: Last-summarised list of errors
 181         _base_warn_lines: Last-summarised list of warnings
 182         _build_period_us: Time taken for a single build (float object).
 183         _complete_delay: Expected delay until completion (timedelta)
 184         _next_delay_update: Next time we plan to display a progress update
 185                 (datatime)
 186         _show_unknown: Show unknown boards (those not built) in summary
 187         _timestamps: List of timestamps for the completion of the last
 188             last _timestamp_count builds. Each is a datetime object.
 189         _timestamp_count: Number of timestamps to keep in our list.
 190         _working_dir: Base working directory containing all threads
 191     """
 192     class Outcome:
 193         """Records a build outcome for a single make invocation
 194
 195         Public Members:
 196             rc: Outcome value (OUTCOME_...)
 197             err_lines: List of error lines or [] if none
 198             sizes: Dictionary of image size information, keyed by filename
 199                 - Each value is itself a dictionary containing
 200                     values for 'text', 'data' and 'bss', being the integer
 201                     size in bytes of each section.
 202             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 203                     value is itself a dictionary:
 204                         key: function name
 205                         value: Size of function in bytes
 206             config: Dictionary keyed by filename - e.g. '.config'. Each
 207                     value is itself a dictionary:
 208                         key: config name
 209                         value: config value
 210             environment: Dictionary keyed by environment variable, Each
 211                      value is the value of environment variable.
 212         """
 213         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 214                      environment):
 215             self.rc = rc
 216             self.err_lines = err_lines
 217             self.sizes = sizes
 218             self.func_sizes = func_sizes
 219             self.config = config
 220             self.environment = environment
 221
 222     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 223                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 224                  no_subdirs=False, full_path=False, verbose_build=False,
 225                  incremental=False, per_board_out_dir=False,
 226                  config_only=False, squash_config_y=False,
 227                  warnings_as_errors=False):
 228         """Create a new Builder object
 229
 230         Args:
 231             toolchains: Toolchains object to use for building
 232             base_dir: Base directory to use for builder
 233             git_dir: Git directory containing source repository
 234             num_threads: Number of builder threads to run
 235             num_jobs: Number of jobs to run at once (passed to make as -j)
 236             gnu_make: the command name of GNU Make.
 237             checkout: True to check out source, False to skip that step.
 238                 This is used for testing.
 239             show_unknown: Show unknown boards (those not built) in summary
 240             step: 1 to process every commit, n to process every nth commit
 241             no_subdirs: Don't create subdirectories when building current
 242                 source for a single board
 243             full_path: Return the full path in CROSS_COMPILE and don't set
 244                 PATH
 245             verbose_build: Run build with V=1 and don't use 'make -s'
 246             incremental: Always perform incremental builds; don't run make
 247                 mrproper when configuring
 248             per_board_out_dir: Build in a separate persistent directory per
 249                 board rather than a thread-specific directory
 250             config_only: Only configure each build, don't build it
 251             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 252             warnings_as_errors: Treat all compiler warnings as errors
 253         """
 254         self.toolchains = toolchains
 255         self.base_dir = base_dir
 256         self._working_dir = os.path.join(base_dir, '.bm-work')
 257         self.threads = []
 258         self.do_make = self.Make
 259         self.gnu_make = gnu_make
 260         self.checkout = checkout
 261         self.num_threads = num_threads
 262         self.num_jobs = num_jobs
 263         self.already_done = 0
 264         self.force_build = False
 265         self.git_dir = git_dir
 266         self._show_unknown = show_unknown
 267         self._timestamp_count = 10
 268         self._build_period_us = None
 269         self._complete_delay = None
 270         self._next_delay_update = datetime.now()
 271         self.force_config_on_failure = True
 272         self.force_build_failures = False
 273         self.force_reconfig = False
 274         self._step = step
 275         self.in_tree = False
 276         self._error_lines = 0
 277         self.no_subdirs = no_subdirs
 278         self.full_path = full_path
 279         self.verbose_build = verbose_build
 280         self.config_only = config_only
 281         self.squash_config_y = squash_config_y
 282         self.config_filenames = BASE_CONFIG_FILENAMES
 283         if not self.squash_config_y:
 284             self.config_filenames += EXTRA_CONFIG_FILENAMES
 285
 286         self.warnings_as_errors = warnings_as_errors
 287         self.col = terminal.Color()
 288
 289         self._re_function = re.compile('(.*): In function.*')
 290         self._re_files = re.compile('In file included from.*')
 291         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 292         self._re_dtb_warning = re.compile('(.*): Warning .*')
 293         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 294
 295         self.queue = queue.Queue()
 296         self.out_queue = queue.Queue()
 297         for i in range(self.num_threads):
 298             t = builderthread.BuilderThread(self, i, incremental,
 299                     per_board_out_dir)
 300             t.setDaemon(True)
 301             t.start()
 302             self.threads.append(t)
 303
 304         self.last_line_len = 0
 305         t = builderthread.ResultThread(self)
 306         t.setDaemon(True)
 307         t.start()
 308         self.threads.append(t)
 309
 310         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 311         self.re_make_err = re.compile('|'.join(ignore_lines))
 312
 313         # Handle existing graceful with SIGINT / Ctrl-C
 314         signal.signal(signal.SIGINT, self.signal_handler)
 315
 316     def __del__(self):
 317         """Get rid of all threads created by the builder"""
 318         for t in self.threads:
 319             del t
 320
 321     def signal_handler(self, signal, frame):
 322         sys.exit(1)
 323
 324     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 325                           show_detail=False, show_bloat=False,
 326                           list_error_boards=False, show_config=False,
 327                           show_environment=False):
 328         """Setup display options for the builder.
 329
 330         show_errors: True to show summarised error/warning info
 331         show_sizes: Show size deltas
 332         show_detail: Show detail for each board
 333         show_bloat: Show detail for each function
 334         list_error_boards: Show the boards which caused each error/warning
 335         show_config: Show config deltas
 336         show_environment: Show environment deltas
 337         """
 338         self._show_errors = show_errors
 339         self._show_sizes = show_sizes
 340         self._show_detail = show_detail
 341         self._show_bloat = show_bloat
 342         self._list_error_boards = list_error_boards
 343         self._show_config = show_config
 344         self._show_environment = show_environment
 345
 346     def _AddTimestamp(self):
 347         """Add a new timestamp to the list and record the build period.
 348
 349         The build period is the length of time taken to perform a single
 350         build (one board, one commit).
 351         """
 352         now = datetime.now()
 353         self._timestamps.append(now)
 354         count = len(self._timestamps)
 355         delta = self._timestamps[-1] - self._timestamps[0]
 356         seconds = delta.total_seconds()
 357
 358         # If we have enough data, estimate build period (time taken for a
 359         # single build) and therefore completion time.
 360         if count > 1 and self._next_delay_update < now:
 361             self._next_delay_update = now + timedelta(seconds=2)
 362             if seconds > 0:
 363                 self._build_period = float(seconds) / count
 364                 todo = self.count - self.upto
 365                 self._complete_delay = timedelta(microseconds=
 366                         self._build_period * todo * 1000000)
 367                 # Round it
 368                 self._complete_delay -= timedelta(
 369                         microseconds=self._complete_delay.microseconds)
 370
 371         if seconds > 60:
 372             self._timestamps.popleft()
 373             count -= 1
 374
 375     def ClearLine(self, length):
 376         """Clear any characters on the current line
 377
 378         Make way for a new line of length 'length', by outputting enough
 379         spaces to clear out the old line. Then remember the new length for
 380         next time.
 381
 382         Args:
 383             length: Length of new line, in characters
 384         """
 385         if length < self.last_line_len:
 386             Print(' ' * (self.last_line_len - length), newline=False)
 387             Print('\r', newline=False)
 388         self.last_line_len = length
 389         sys.stdout.flush()
 390
 391     def SelectCommit(self, commit, checkout=True):
 392         """Checkout the selected commit for this build
 393         """
 394         self.commit = commit
 395         if checkout and self.checkout:
 396             gitutil.Checkout(commit.hash)
 397
 398     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 399         """Run make
 400
 401         Args:
 402             commit: Commit object that is being built
 403             brd: Board object that is being built
 404             stage: Stage that we are at (mrproper, config, build)
 405             cwd: Directory where make should be run
 406             args: Arguments to pass to make
 407             kwargs: Arguments to pass to command.RunPipe()
 408         """
 409         cmd = [self.gnu_make] + list(args)
 410         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 411                 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
 412         if self.verbose_build:
 413             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 414             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 415         return result
 416
 417     def ProcessResult(self, result):
 418         """Process the result of a build, showing progress information
 419
 420         Args:
 421             result: A CommandResult object, which indicates the result for
 422                     a single build
 423         """
 424         col = terminal.Color()
 425         if result:
 426             target = result.brd.target
 427
 428             self.upto += 1
 429             if result.return_code != 0:
 430                 self.fail += 1
 431             elif result.stderr:
 432                 self.warned += 1
 433             if result.already_done:
 434                 self.already_done += 1
 435             if self._verbose:
 436                 Print('\r', newline=False)
 437                 self.ClearLine(0)
 438                 boards_selected = {target : result.brd}
 439                 self.ResetResultSummary(boards_selected)
 440                 self.ProduceResultSummary(result.commit_upto, self.commits,
 441                                           boards_selected)
 442         else:
 443             target = '(starting)'
 444
 445         # Display separate counts for ok, warned and fail
 446         ok = self.upto - self.warned - self.fail
 447         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 448         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 449         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 450
 451         name = ' /%-5d  ' % self.count
 452
 453         # Add our current completion time estimate
 454         self._AddTimestamp()
 455         if self._complete_delay:
 456             name += '%s  : ' % self._complete_delay
 457         # When building all boards for a commit, we can print a commit
 458         # progress message.
 459         if result and result.commit_upto is None:
 460             name += 'commit %2d/%-3d' % (self.commit_upto + 1,
 461                     self.commit_count)
 462
 463         name += target
 464         Print(line + name, newline=False)
 465         length = 16 + len(name)
 466         self.ClearLine(length)
 467
 468     def _GetOutputDir(self, commit_upto):
 469         """Get the name of the output directory for a commit number
 470
 471         The output directory is typically .../<branch>/<commit>.
 472
 473         Args:
 474             commit_upto: Commit number to use (0..self.count-1)
 475         """
 476         commit_dir = None
 477         if self.commits:
 478             commit = self.commits[commit_upto]
 479             subject = commit.subject.translate(trans_valid_chars)
 480             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
 481                     self.commit_count, commit.hash, subject[:20]))
 482         elif not self.no_subdirs:
 483             commit_dir = 'current'
 484         if not commit_dir:
 485             return self.base_dir
 486         return os.path.join(self.base_dir, commit_dir)
 487
 488     def GetBuildDir(self, commit_upto, target):
 489         """Get the name of the build directory for a commit number
 490
 491         The build directory is typically .../<branch>/<commit>/<target>.
 492
 493         Args:
 494             commit_upto: Commit number to use (0..self.count-1)
 495             target: Target name
 496         """
 497         output_dir = self._GetOutputDir(commit_upto)
 498         return os.path.join(output_dir, target)
 499
 500     def GetDoneFile(self, commit_upto, target):
 501         """Get the name of the done file for a commit number
 502
 503         Args:
 504             commit_upto: Commit number to use (0..self.count-1)
 505             target: Target name
 506         """
 507         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 508
 509     def GetSizesFile(self, commit_upto, target):
 510         """Get the name of the sizes file for a commit number
 511
 512         Args:
 513             commit_upto: Commit number to use (0..self.count-1)
 514             target: Target name
 515         """
 516         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 517
 518     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 519         """Get the name of the funcsizes file for a commit number and ELF file
 520
 521         Args:
 522             commit_upto: Commit number to use (0..self.count-1)
 523             target: Target name
 524             elf_fname: Filename of elf image
 525         """
 526         return os.path.join(self.GetBuildDir(commit_upto, target),
 527                             '%s.sizes' % elf_fname.replace('/', '-'))
 528
 529     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 530         """Get the name of the objdump file for a commit number and ELF file
 531
 532         Args:
 533             commit_upto: Commit number to use (0..self.count-1)
 534             target: Target name
 535             elf_fname: Filename of elf image
 536         """
 537         return os.path.join(self.GetBuildDir(commit_upto, target),
 538                             '%s.objdump' % elf_fname.replace('/', '-'))
 539
 540     def GetErrFile(self, commit_upto, target):
 541         """Get the name of the err file for a commit number
 542
 543         Args:
 544             commit_upto: Commit number to use (0..self.count-1)
 545             target: Target name
 546         """
 547         output_dir = self.GetBuildDir(commit_upto, target)
 548         return os.path.join(output_dir, 'err')
 549
 550     def FilterErrors(self, lines):
 551         """Filter out errors in which we have no interest
 552
 553         We should probably use map().
 554
 555         Args:
 556             lines: List of error lines, each a string
 557         Returns:
 558             New list with only interesting lines included
 559         """
 560         out_lines = []
 561         for line in lines:
 562             if not self.re_make_err.search(line):
 563                 out_lines.append(line)
 564         return out_lines
 565
 566     def ReadFuncSizes(self, fname, fd):
 567         """Read function sizes from the output of 'nm'
 568
 569         Args:
 570             fd: File containing data to read
 571             fname: Filename we are reading from (just for errors)
 572
 573         Returns:
 574             Dictionary containing size of each function in bytes, indexed by
 575             function name.
 576         """
 577         sym = {}
 578         for line in fd.readlines():
 579             try:
 580                 size, type, name = line[:-1].split()
 581             except:
 582                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 583                 continue
 584             if type in 'tTdDbB':
 585                 # function names begin with '.' on 64-bit powerpc
 586                 if '.' in name[1:]:
 587                     name = 'static.' + name.split('.')[0]
 588                 sym[name] = sym.get(name, 0) + int(size, 16)
 589         return sym
 590
 591     def _ProcessConfig(self, fname):
 592         """Read in a .config, autoconf.mk or autoconf.h file
 593
 594         This function handles all config file types. It ignores comments and
 595         any #defines which don't start with CONFIG_.
 596
 597         Args:
 598             fname: Filename to read
 599
 600         Returns:
 601             Dictionary:
 602                 key: Config name (e.g. CONFIG_DM)
 603                 value: Config value (e.g. 1)
 604         """
 605         config = {}
 606         if os.path.exists(fname):
 607             with open(fname) as fd:
 608                 for line in fd:
 609                     line = line.strip()
 610                     if line.startswith('#define'):
 611                         values = line[8:].split(' ', 1)
 612                         if len(values) > 1:
 613                             key, value = values
 614                         else:
 615                             key = values[0]
 616                             value = '1' if self.squash_config_y else ''
 617                         if not key.startswith('CONFIG_'):
 618                             continue
 619                     elif not line or line[0] in ['#', '*', '/']:
 620                         continue
 621                     else:
 622                         key, value = line.split('=', 1)
 623                     if self.squash_config_y and value == 'y':
 624                         value = '1'
 625                     config[key] = value
 626         return config
 627
 628     def _ProcessEnvironment(self, fname):
 629         """Read in a uboot.env file
 630
 631         This function reads in environment variables from a file.
 632
 633         Args:
 634             fname: Filename to read
 635
 636         Returns:
 637             Dictionary:
 638                 key: environment variable (e.g. bootlimit)
 639                 value: value of environment variable (e.g. 1)
 640         """
 641         environment = {}
 642         if os.path.exists(fname):
 643             with open(fname) as fd:
 644                 for line in fd.read().split('\0'):
 645                     try:
 646                         key, value = line.split('=', 1)
 647                         environment[key] = value
 648                     except ValueError:
 649                         # ignore lines we can't parse
 650                         pass
 651         return environment
 652
 653     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 654                         read_config, read_environment):
 655         """Work out the outcome of a build.
 656
 657         Args:
 658             commit_upto: Commit number to check (0..n-1)
 659             target: Target board to check
 660             read_func_sizes: True to read function size information
 661             read_config: True to read .config and autoconf.h files
 662             read_environment: True to read uboot.env files
 663
 664         Returns:
 665             Outcome object
 666         """
 667         done_file = self.GetDoneFile(commit_upto, target)
 668         sizes_file = self.GetSizesFile(commit_upto, target)
 669         sizes = {}
 670         func_sizes = {}
 671         config = {}
 672         environment = {}
 673         if os.path.exists(done_file):
 674             with open(done_file, 'r') as fd:
 675                 try:
 676                     return_code = int(fd.readline())
 677                 except ValueError:
 678                     # The file may be empty due to running out of disk space.
 679                     # Try a rebuild
 680                     return_code = 1
 681                 err_lines = []
 682                 err_file = self.GetErrFile(commit_upto, target)
 683                 if os.path.exists(err_file):
 684                     with open(err_file, 'r') as fd:
 685                         err_lines = self.FilterErrors(fd.readlines())
 686
 687                 # Decide whether the build was ok, failed or created warnings
 688                 if return_code:
 689                     rc = OUTCOME_ERROR
 690                 elif len(err_lines):
 691                     rc = OUTCOME_WARNING
 692                 else:
 693                     rc = OUTCOME_OK
 694
 695                 # Convert size information to our simple format
 696                 if os.path.exists(sizes_file):
 697                     with open(sizes_file, 'r') as fd:
 698                         for line in fd.readlines():
 699                             values = line.split()
 700                             rodata = 0
 701                             if len(values) > 6:
 702                                 rodata = int(values[6], 16)
 703                             size_dict = {
 704                                 'all' : int(values[0]) + int(values[1]) +
 705                                         int(values[2]),
 706                                 'text' : int(values[0]) - rodata,
 707                                 'data' : int(values[1]),
 708                                 'bss' : int(values[2]),
 709                                 'rodata' : rodata,
 710                             }
 711                             sizes[values[5]] = size_dict
 712
 713             if read_func_sizes:
 714                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 715                 for fname in glob.glob(pattern):
 716                     with open(fname, 'r') as fd:
 717                         dict_name = os.path.basename(fname).replace('.sizes',
 718                                                                     '')
 719                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 720
 721             if read_config:
 722                 output_dir = self.GetBuildDir(commit_upto, target)
 723                 for name in self.config_filenames:
 724                     fname = os.path.join(output_dir, name)
 725                     config[name] = self._ProcessConfig(fname)
 726
 727             if read_environment:
 728                 output_dir = self.GetBuildDir(commit_upto, target)
 729                 fname = os.path.join(output_dir, 'uboot.env')
 730                 environment = self._ProcessEnvironment(fname)
 731
 732             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 733                                    environment)
 734
 735         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 736
 737     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 738                          read_config, read_environment):
 739         """Calculate a summary of the results of building a commit.
 740
 741         Args:
 742             board_selected: Dict containing boards to summarise
 743             commit_upto: Commit number to summarize (0..self.count-1)
 744             read_func_sizes: True to read function size information
 745             read_config: True to read .config and autoconf.h files
 746             read_environment: True to read uboot.env files
 747
 748         Returns:
 749             Tuple:
 750                 Dict containing boards which passed building this commit.
 751                     keyed by board.target
 752                 List containing a summary of error lines
 753                 Dict keyed by error line, containing a list of the Board
 754                     objects with that error
 755                 List containing a summary of warning lines
 756                 Dict keyed by error line, containing a list of the Board
 757                     objects with that warning
 758                 Dictionary keyed by board.target. Each value is a dictionary:
 759                     key: filename - e.g. '.config'
 760                     value is itself a dictionary:
 761                         key: config name
 762                         value: config value
 763                 Dictionary keyed by board.target. Each value is a dictionary:
 764                     key: environment variable
 765                     value: value of environment variable
 766         """
 767         def AddLine(lines_summary, lines_boards, line, board):
 768             line = line.rstrip()
 769             if line in lines_boards:
 770                 lines_boards[line].append(board)
 771             else:
 772                 lines_boards[line] = [board]
 773                 lines_summary.append(line)
 774
 775         board_dict = {}
 776         err_lines_summary = []
 777         err_lines_boards = {}
 778         warn_lines_summary = []
 779         warn_lines_boards = {}
 780         config = {}
 781         environment = {}
 782
 783         for board in boards_selected.values():
 784             outcome = self.GetBuildOutcome(commit_upto, board.target,
 785                                            read_func_sizes, read_config,
 786                                            read_environment)
 787             board_dict[board.target] = outcome
 788             last_func = None
 789             last_was_warning = False
 790             for line in outcome.err_lines:
 791                 if line:
 792                     if (self._re_function.match(line) or
 793                             self._re_files.match(line)):
 794                         last_func = line
 795                     else:
 796                         is_warning = (self._re_warning.match(line) or
 797                                       self._re_dtb_warning.match(line))
 798                         is_note = self._re_note.match(line)
 799                         if is_warning or (last_was_warning and is_note):
 800                             if last_func:
 801                                 AddLine(warn_lines_summary, warn_lines_boards,
 802                                         last_func, board)
 803                             AddLine(warn_lines_summary, warn_lines_boards,
 804                                     line, board)
 805                         else:
 806                             if last_func:
 807                                 AddLine(err_lines_summary, err_lines_boards,
 808                                         last_func, board)
 809                             AddLine(err_lines_summary, err_lines_boards,
 810                                     line, board)
 811                         last_was_warning = is_warning
 812                         last_func = None
 813             tconfig = Config(self.config_filenames, board.target)
 814             for fname in self.config_filenames:
 815                 if outcome.config:
 816                     for key, value in outcome.config[fname].items():
 817                         tconfig.Add(fname, key, value)
 818             config[board.target] = tconfig
 819
 820             tenvironment = Environment(board.target)
 821             if outcome.environment:
 822                 for key, value in outcome.environment.items():
 823                     tenvironment.Add(key, value)
 824             environment[board.target] = tenvironment
 825
 826         return (board_dict, err_lines_summary, err_lines_boards,
 827                 warn_lines_summary, warn_lines_boards, config, environment)
 828
 829     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 830         """Add an output to our list of outcomes for each architecture
 831
 832         This simple function adds failing boards (changes) to the
 833         relevant architecture string, so we can print the results out
 834         sorted by architecture.
 835
 836         Args:
 837              board_dict: Dict containing all boards
 838              arch_list: Dict keyed by arch name. Value is a string containing
 839                     a list of board names which failed for that arch.
 840              changes: List of boards to add to arch_list
 841              color: terminal.Colour object
 842         """
 843         done_arch = {}
 844         for target in changes:
 845             if target in board_dict:
 846                 arch = board_dict[target].arch
 847             else:
 848                 arch = 'unknown'
 849             str = self.col.Color(color, ' ' + target)
 850             if not arch in done_arch:
 851                 str = ' %s  %s' % (self.col.Color(color, char), str)
 852                 done_arch[arch] = True
 853             if not arch in arch_list:
 854                 arch_list[arch] = str
 855             else:
 856                 arch_list[arch] += str
 857
 858
 859     def ColourNum(self, num):
 860         color = self.col.RED if num > 0 else self.col.GREEN
 861         if num == 0:
 862             return '0'
 863         return self.col.Color(color, str(num))
 864
 865     def ResetResultSummary(self, board_selected):
 866         """Reset the results summary ready for use.
 867
 868         Set up the base board list to be all those selected, and set the
 869         error lines to empty.
 870
 871         Following this, calls to PrintResultSummary() will use this
 872         information to work out what has changed.
 873
 874         Args:
 875             board_selected: Dict containing boards to summarise, keyed by
 876                 board.target
 877         """
 878         self._base_board_dict = {}
 879         for board in board_selected:
 880             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 881                                                            {})
 882         self._base_err_lines = []
 883         self._base_warn_lines = []
 884         self._base_err_line_boards = {}
 885         self._base_warn_line_boards = {}
 886         self._base_config = None
 887         self._base_environment = None
 888
 889     def PrintFuncSizeDetail(self, fname, old, new):
 890         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 891         delta, common = [], {}
 892
 893         for a in old:
 894             if a in new:
 895                 common[a] = 1
 896
 897         for name in old:
 898             if name not in common:
 899                 remove += 1
 900                 down += old[name]
 901                 delta.append([-old[name], name])
 902
 903         for name in new:
 904             if name not in common:
 905                 add += 1
 906                 up += new[name]
 907                 delta.append([new[name], name])
 908
 909         for name in common:
 910                 diff = new.get(name, 0) - old.get(name, 0)
 911                 if diff > 0:
 912                     grow, up = grow + 1, up + diff
 913                 elif diff < 0:
 914                     shrink, down = shrink + 1, down - diff
 915                 delta.append([diff, name])
 916
 917         delta.sort()
 918         delta.reverse()
 919
 920         args = [add, -remove, grow, -shrink, up, -down, up - down]
 921         if max(args) == 0 and min(args) == 0:
 922             return
 923         args = [self.ColourNum(x) for x in args]
 924         indent = ' ' * 15
 925         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
 926               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
 927         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
 928                                          'delta'))
 929         for diff, name in delta:
 930             if diff:
 931                 color = self.col.RED if diff > 0 else self.col.GREEN
 932                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
 933                         old.get(name, '-'), new.get(name,'-'), diff)
 934                 Print(msg, colour=color)
 935
 936
 937     def PrintSizeDetail(self, target_list, show_bloat):
 938         """Show details size information for each board
 939
 940         Args:
 941             target_list: List of targets, each a dict containing:
 942                     'target': Target name
 943                     'total_diff': Total difference in bytes across all areas
 944                     <part_name>: Difference for that part
 945             show_bloat: Show detail for each function
 946         """
 947         targets_by_diff = sorted(target_list, reverse=True,
 948         key=lambda x: x['_total_diff'])
 949         for result in targets_by_diff:
 950             printed_target = False
 951             for name in sorted(result):
 952                 diff = result[name]
 953                 if name.startswith('_'):
 954                     continue
 955                 if diff != 0:
 956                     color = self.col.RED if diff > 0 else self.col.GREEN
 957                 msg = ' %s %+d' % (name, diff)
 958                 if not printed_target:
 959                     Print('%10s  %-15s:' % ('', result['_target']),
 960                           newline=False)
 961                     printed_target = True
 962                 Print(msg, colour=color, newline=False)
 963             if printed_target:
 964                 Print()
 965                 if show_bloat:
 966                     target = result['_target']
 967                     outcome = result['_outcome']
 968                     base_outcome = self._base_board_dict[target]
 969                     for fname in outcome.func_sizes:
 970                         self.PrintFuncSizeDetail(fname,
 971                                                  base_outcome.func_sizes[fname],
 972                                                  outcome.func_sizes[fname])
 973
 974
 975     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
 976                          show_bloat):
 977         """Print a summary of image sizes broken down by section.
 978
 979         The summary takes the form of one line per architecture. The
 980         line contains deltas for each of the sections (+ means the section
 981         got bigger, - means smaller). The nunmbers are the average number
 982         of bytes that a board in this section increased by.
 983
 984         For example:
 985            powerpc: (622 boards)   text -0.0
 986           arm: (285 boards)   text -0.0
 987           nds32: (3 boards)   text -8.0
 988
 989         Args:
 990             board_selected: Dict containing boards to summarise, keyed by
 991                 board.target
 992             board_dict: Dict containing boards for which we built this
 993                 commit, keyed by board.target. The value is an Outcome object.
 994             show_detail: Show detail for each board
 995             show_bloat: Show detail for each function
 996         """
 997         arch_list = {}
 998         arch_count = {}
 999
1000         # Calculate changes in size for different image parts
1001         # The previous sizes are in Board.sizes, for each board
1002         for target in board_dict:
1003             if target not in board_selected:
1004                 continue
1005             base_sizes = self._base_board_dict[target].sizes
1006             outcome = board_dict[target]
1007             sizes = outcome.sizes
1008
1009             # Loop through the list of images, creating a dict of size
1010             # changes for each image/part. We end up with something like
1011             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1012             # which means that U-Boot data increased by 5 bytes and SPL
1013             # text decreased by 4.
1014             err = {'_target' : target}
1015             for image in sizes:
1016                 if image in base_sizes:
1017                     base_image = base_sizes[image]
1018                     # Loop through the text, data, bss parts
1019                     for part in sorted(sizes[image]):
1020                         diff = sizes[image][part] - base_image[part]
1021                         col = None
1022                         if diff:
1023                             if image == 'u-boot':
1024                                 name = part
1025                             else:
1026                                 name = image + ':' + part
1027                             err[name] = diff
1028             arch = board_selected[target].arch
1029             if not arch in arch_count:
1030                 arch_count[arch] = 1
1031             else:
1032                 arch_count[arch] += 1
1033             if not sizes:
1034                 pass    # Only add to our list when we have some stats
1035             elif not arch in arch_list:
1036                 arch_list[arch] = [err]
1037             else:
1038                 arch_list[arch].append(err)
1039
1040         # We now have a list of image size changes sorted by arch
1041         # Print out a summary of these
1042         for arch, target_list in arch_list.items():
1043             # Get total difference for each type
1044             totals = {}
1045             for result in target_list:
1046                 total = 0
1047                 for name, diff in result.items():
1048                     if name.startswith('_'):
1049                         continue
1050                     total += diff
1051                     if name in totals:
1052                         totals[name] += diff
1053                     else:
1054                         totals[name] = diff
1055                 result['_total_diff'] = total
1056                 result['_outcome'] = board_dict[result['_target']]
1057
1058             count = len(target_list)
1059             printed_arch = False
1060             for name in sorted(totals):
1061                 diff = totals[name]
1062                 if diff:
1063                     # Display the average difference in this name for this
1064                     # architecture
1065                     avg_diff = float(diff) / count
1066                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1067                     msg = ' %s %+1.1f' % (name, avg_diff)
1068                     if not printed_arch:
1069                         Print('%10s: (for %d/%d boards)' % (arch, count,
1070                               arch_count[arch]), newline=False)
1071                         printed_arch = True
1072                     Print(msg, colour=color, newline=False)
1073
1074             if printed_arch:
1075                 Print()
1076                 if show_detail:
1077                     self.PrintSizeDetail(target_list, show_bloat)
1078
1079
1080     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1081                            err_line_boards, warn_lines, warn_line_boards,
1082                            config, environment, show_sizes, show_detail,
1083                            show_bloat, show_config, show_environment):
1084         """Compare results with the base results and display delta.
1085
1086         Only boards mentioned in board_selected will be considered. This
1087         function is intended to be called repeatedly with the results of
1088         each commit. It therefore shows a 'diff' between what it saw in
1089         the last call and what it sees now.
1090
1091         Args:
1092             board_selected: Dict containing boards to summarise, keyed by
1093                 board.target
1094             board_dict: Dict containing boards for which we built this
1095                 commit, keyed by board.target. The value is an Outcome object.
1096             err_lines: A list of errors for this commit, or [] if there is
1097                 none, or we don't want to print errors
1098             err_line_boards: Dict keyed by error line, containing a list of
1099                 the Board objects with that error
1100             warn_lines: A list of warnings for this commit, or [] if there is
1101                 none, or we don't want to print errors
1102             warn_line_boards: Dict keyed by warning line, containing a list of
1103                 the Board objects with that warning
1104             config: Dictionary keyed by filename - e.g. '.config'. Each
1105                     value is itself a dictionary:
1106                         key: config name
1107                         value: config value
1108             environment: Dictionary keyed by environment variable, Each
1109                      value is the value of environment variable.
1110             show_sizes: Show image size deltas
1111             show_detail: Show detail for each board
1112             show_bloat: Show detail for each function
1113             show_config: Show config changes
1114             show_environment: Show environment changes
1115         """
1116         def _BoardList(line, line_boards):
1117             """Helper function to get a line of boards containing a line
1118
1119             Args:
1120                 line: Error line to search for
1121             Return:
1122                 String containing a list of boards with that error line, or
1123                 '' if the user has not requested such a list
1124             """
1125             if self._list_error_boards:
1126                 names = []
1127                 for board in line_boards[line]:
1128                     if not board.target in names:
1129                         names.append(board.target)
1130                 names_str = '(%s) ' % ','.join(names)
1131             else:
1132                 names_str = ''
1133             return names_str
1134
1135         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1136                             char):
1137             better_lines = []
1138             worse_lines = []
1139             for line in lines:
1140                 if line not in base_lines:
1141                     worse_lines.append(char + '+' +
1142                             _BoardList(line, line_boards) + line)
1143             for line in base_lines:
1144                 if line not in lines:
1145                     better_lines.append(char + '-' +
1146                             _BoardList(line, base_line_boards) + line)
1147             return better_lines, worse_lines
1148
1149         def _CalcConfig(delta, name, config):
1150             """Calculate configuration changes
1151
1152             Args:
1153                 delta: Type of the delta, e.g. '+'
1154                 name: name of the file which changed (e.g. .config)
1155                 config: configuration change dictionary
1156                     key: config name
1157                     value: config value
1158             Returns:
1159                 String containing the configuration changes which can be
1160                     printed
1161             """
1162             out = ''
1163             for key in sorted(config.keys()):
1164                 out += '%s=%s ' % (key, config[key])
1165             return '%s %s: %s' % (delta, name, out)
1166
1167         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1168             """Add changes in configuration to a list
1169
1170             Args:
1171                 lines: list to add to
1172                 name: config file name
1173                 config_plus: configurations added, dictionary
1174                     key: config name
1175                     value: config value
1176                 config_minus: configurations removed, dictionary
1177                     key: config name
1178                     value: config value
1179                 config_change: configurations changed, dictionary
1180                     key: config name
1181                     value: config value
1182             """
1183             if config_plus:
1184                 lines.append(_CalcConfig('+', name, config_plus))
1185             if config_minus:
1186                 lines.append(_CalcConfig('-', name, config_minus))
1187             if config_change:
1188                 lines.append(_CalcConfig('c', name, config_change))
1189
1190         def _OutputConfigInfo(lines):
1191             for line in lines:
1192                 if not line:
1193                     continue
1194                 if line[0] == '+':
1195                     col = self.col.GREEN
1196                 elif line[0] == '-':
1197                     col = self.col.RED
1198                 elif line[0] == 'c':
1199                     col = self.col.YELLOW
1200                 Print('   ' + line, newline=True, colour=col)
1201
1202
1203         ok_boards = []      # List of boards fixed since last commit
1204         warn_boards = []    # List of boards with warnings since last commit
1205         err_boards = []     # List of new broken boards since last commit
1206         new_boards = []     # List of boards that didn't exist last time
1207         unknown_boards = [] # List of boards that were not built
1208
1209         for target in board_dict:
1210             if target not in board_selected:
1211                 continue
1212
1213             # If the board was built last time, add its outcome to a list
1214             if target in self._base_board_dict:
1215                 base_outcome = self._base_board_dict[target].rc
1216                 outcome = board_dict[target]
1217                 if outcome.rc == OUTCOME_UNKNOWN:
1218                     unknown_boards.append(target)
1219                 elif outcome.rc < base_outcome:
1220                     if outcome.rc == OUTCOME_WARNING:
1221                         warn_boards.append(target)
1222                     else:
1223                         ok_boards.append(target)
1224                 elif outcome.rc > base_outcome:
1225                     if outcome.rc == OUTCOME_WARNING:
1226                         warn_boards.append(target)
1227                     else:
1228                         err_boards.append(target)
1229             else:
1230                 new_boards.append(target)
1231
1232         # Get a list of errors that have appeared, and disappeared
1233         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1234                 self._base_err_line_boards, err_lines, err_line_boards, '')
1235         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1236                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1237
1238         # Display results by arch
1239         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1240                 worse_err, better_err, worse_warn, better_warn)):
1241             arch_list = {}
1242             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1243                     self.col.GREEN)
1244             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1245                     self.col.YELLOW)
1246             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1247                     self.col.RED)
1248             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1249             if self._show_unknown:
1250                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1251                         self.col.MAGENTA)
1252             for arch, target_list in arch_list.items():
1253                 Print('%10s: %s' % (arch, target_list))
1254                 self._error_lines += 1
1255             if better_err:
1256                 Print('\n'.join(better_err), colour=self.col.GREEN)
1257                 self._error_lines += 1
1258             if worse_err:
1259                 Print('\n'.join(worse_err), colour=self.col.RED)
1260                 self._error_lines += 1
1261             if better_warn:
1262                 Print('\n'.join(better_warn), colour=self.col.CYAN)
1263                 self._error_lines += 1
1264             if worse_warn:
1265                 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
1266                 self._error_lines += 1
1267
1268         if show_sizes:
1269             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1270                                   show_bloat)
1271
1272         if show_environment and self._base_environment:
1273             lines = []
1274
1275             for target in board_dict:
1276                 if target not in board_selected:
1277                     continue
1278
1279                 tbase = self._base_environment[target]
1280                 tenvironment = environment[target]
1281                 environment_plus = {}
1282                 environment_minus = {}
1283                 environment_change = {}
1284                 base = tbase.environment
1285                 for key, value in tenvironment.environment.items():
1286                     if key not in base:
1287                         environment_plus[key] = value
1288                 for key, value in base.items():
1289                     if key not in tenvironment.environment:
1290                         environment_minus[key] = value
1291                 for key, value in base.items():
1292                     new_value = tenvironment.environment.get(key)
1293                     if new_value and value != new_value:
1294                         desc = '%s -> %s' % (value, new_value)
1295                         environment_change[key] = desc
1296
1297                 _AddConfig(lines, target, environment_plus, environment_minus,
1298                            environment_change)
1299
1300             _OutputConfigInfo(lines)
1301
1302         if show_config and self._base_config:
1303             summary = {}
1304             arch_config_plus = {}
1305             arch_config_minus = {}
1306             arch_config_change = {}
1307             arch_list = []
1308
1309             for target in board_dict:
1310                 if target not in board_selected:
1311                     continue
1312                 arch = board_selected[target].arch
1313                 if arch not in arch_list:
1314                     arch_list.append(arch)
1315
1316             for arch in arch_list:
1317                 arch_config_plus[arch] = {}
1318                 arch_config_minus[arch] = {}
1319                 arch_config_change[arch] = {}
1320                 for name in self.config_filenames:
1321                     arch_config_plus[arch][name] = {}
1322                     arch_config_minus[arch][name] = {}
1323                     arch_config_change[arch][name] = {}
1324
1325             for target in board_dict:
1326                 if target not in board_selected:
1327                     continue
1328
1329                 arch = board_selected[target].arch
1330
1331                 all_config_plus = {}
1332                 all_config_minus = {}
1333                 all_config_change = {}
1334                 tbase = self._base_config[target]
1335                 tconfig = config[target]
1336                 lines = []
1337                 for name in self.config_filenames:
1338                     if not tconfig.config[name]:
1339                         continue
1340                     config_plus = {}
1341                     config_minus = {}
1342                     config_change = {}
1343                     base = tbase.config[name]
1344                     for key, value in tconfig.config[name].items():
1345                         if key not in base:
1346                             config_plus[key] = value
1347                             all_config_plus[key] = value
1348                     for key, value in base.items():
1349                         if key not in tconfig.config[name]:
1350                             config_minus[key] = value
1351                             all_config_minus[key] = value
1352                     for key, value in base.items():
1353                         new_value = tconfig.config.get(key)
1354                         if new_value and value != new_value:
1355                             desc = '%s -> %s' % (value, new_value)
1356                             config_change[key] = desc
1357                             all_config_change[key] = desc
1358
1359                     arch_config_plus[arch][name].update(config_plus)
1360                     arch_config_minus[arch][name].update(config_minus)
1361                     arch_config_change[arch][name].update(config_change)
1362
1363                     _AddConfig(lines, name, config_plus, config_minus,
1364                                config_change)
1365                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1366                            all_config_change)
1367                 summary[target] = '\n'.join(lines)
1368
1369             lines_by_target = {}
1370             for target, lines in summary.items():
1371                 if lines in lines_by_target:
1372                     lines_by_target[lines].append(target)
1373                 else:
1374                     lines_by_target[lines] = [target]
1375
1376             for arch in arch_list:
1377                 lines = []
1378                 all_plus = {}
1379                 all_minus = {}
1380                 all_change = {}
1381                 for name in self.config_filenames:
1382                     all_plus.update(arch_config_plus[arch][name])
1383                     all_minus.update(arch_config_minus[arch][name])
1384                     all_change.update(arch_config_change[arch][name])
1385                     _AddConfig(lines, name, arch_config_plus[arch][name],
1386                                arch_config_minus[arch][name],
1387                                arch_config_change[arch][name])
1388                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1389                 #arch_summary[target] = '\n'.join(lines)
1390                 if lines:
1391                     Print('%s:' % arch)
1392                     _OutputConfigInfo(lines)
1393
1394             for lines, targets in lines_by_target.items():
1395                 if not lines:
1396                     continue
1397                 Print('%s :' % ' '.join(sorted(targets)))
1398                 _OutputConfigInfo(lines.split('\n'))
1399
1400
1401         # Save our updated information for the next call to this function
1402         self._base_board_dict = board_dict
1403         self._base_err_lines = err_lines
1404         self._base_warn_lines = warn_lines
1405         self._base_err_line_boards = err_line_boards
1406         self._base_warn_line_boards = warn_line_boards
1407         self._base_config = config
1408         self._base_environment = environment
1409
1410         # Get a list of boards that did not get built, if needed
1411         not_built = []
1412         for board in board_selected:
1413             if not board in board_dict:
1414                 not_built.append(board)
1415         if not_built:
1416             Print("Boards not built (%d): %s" % (len(not_built),
1417                   ', '.join(not_built)))
1418
1419     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1420             (board_dict, err_lines, err_line_boards, warn_lines,
1421              warn_line_boards, config, environment) = self.GetResultSummary(
1422                     board_selected, commit_upto,
1423                     read_func_sizes=self._show_bloat,
1424                     read_config=self._show_config,
1425                     read_environment=self._show_environment)
1426             if commits:
1427                 msg = '%02d: %s' % (commit_upto + 1,
1428                         commits[commit_upto].subject)
1429                 Print(msg, colour=self.col.BLUE)
1430             self.PrintResultSummary(board_selected, board_dict,
1431                     err_lines if self._show_errors else [], err_line_boards,
1432                     warn_lines if self._show_errors else [], warn_line_boards,
1433                     config, environment, self._show_sizes, self._show_detail,
1434                     self._show_bloat, self._show_config, self._show_environment)
1435
1436     def ShowSummary(self, commits, board_selected):
1437         """Show a build summary for U-Boot for a given board list.
1438
1439         Reset the result summary, then repeatedly call GetResultSummary on
1440         each commit's results, then display the differences we see.
1441
1442         Args:
1443             commit: Commit objects to summarise
1444             board_selected: Dict containing boards to summarise
1445         """
1446         self.commit_count = len(commits) if commits else 1
1447         self.commits = commits
1448         self.ResetResultSummary(board_selected)
1449         self._error_lines = 0
1450
1451         for commit_upto in range(0, self.commit_count, self._step):
1452             self.ProduceResultSummary(commit_upto, commits, board_selected)
1453         if not self._error_lines:
1454             Print('(no errors to report)', colour=self.col.GREEN)
1455
1456
1457     def SetupBuild(self, board_selected, commits):
1458         """Set up ready to start a build.
1459
1460         Args:
1461             board_selected: Selected boards to build
1462             commits: Selected commits to build
1463         """
1464         # First work out how many commits we will build
1465         count = (self.commit_count + self._step - 1) // self._step
1466         self.count = len(board_selected) * count
1467         self.upto = self.warned = self.fail = 0
1468         self._timestamps = collections.deque()
1469
1470     def GetThreadDir(self, thread_num):
1471         """Get the directory path to the working dir for a thread.
1472
1473         Args:
1474             thread_num: Number of thread to check.
1475         """
1476         return os.path.join(self._working_dir, '%02d' % thread_num)
1477
1478     def _PrepareThread(self, thread_num, setup_git):
1479         """Prepare the working directory for a thread.
1480
1481         This clones or fetches the repo into the thread's work directory.
1482
1483         Args:
1484             thread_num: Thread number (0, 1, ...)
1485             setup_git: True to set up a git repo clone
1486         """
1487         thread_dir = self.GetThreadDir(thread_num)
1488         builderthread.Mkdir(thread_dir)
1489         git_dir = os.path.join(thread_dir, '.git')
1490
1491         # Clone the repo if it doesn't already exist
1492         # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1493         # we have a private index but uses the origin repo's contents?
1494         if setup_git and self.git_dir:
1495             src_dir = os.path.abspath(self.git_dir)
1496             if os.path.exists(git_dir):
1497                 gitutil.Fetch(git_dir, thread_dir)
1498             else:
1499                 Print('\rCloning repo for thread %d' % thread_num,
1500                       newline=False)
1501                 gitutil.Clone(src_dir, thread_dir)
1502                 Print('\r%s\r' % (' ' * 30), newline=False)
1503
1504     def _PrepareWorkingSpace(self, max_threads, setup_git):
1505         """Prepare the working directory for use.
1506
1507         Set up the git repo for each thread.
1508
1509         Args:
1510             max_threads: Maximum number of threads we expect to need.
1511             setup_git: True to set up a git repo clone
1512         """
1513         builderthread.Mkdir(self._working_dir)
1514         for thread in range(max_threads):
1515             self._PrepareThread(thread, setup_git)
1516
1517     def _PrepareOutputSpace(self):
1518         """Get the output directories ready to receive files.
1519
1520         We delete any output directories which look like ones we need to
1521         create. Having left over directories is confusing when the user wants
1522         to check the output manually.
1523         """
1524         if not self.commits:
1525             return
1526         dir_list = []
1527         for commit_upto in range(self.commit_count):
1528             dir_list.append(self._GetOutputDir(commit_upto))
1529
1530         to_remove = []
1531         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1532             if dirname not in dir_list:
1533                 to_remove.append(dirname)
1534         if to_remove:
1535             Print('Removing %d old build directories' % len(to_remove),
1536                   newline=False)
1537             for dirname in to_remove:
1538                 shutil.rmtree(dirname)
1539
1540     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1541         """Build all commits for a list of boards
1542
1543         Args:
1544             commits: List of commits to be build, each a Commit object
1545             boards_selected: Dict of selected boards, key is target name,
1546                     value is Board object
1547             keep_outputs: True to save build output files
1548             verbose: Display build results as they are completed
1549         Returns:
1550             Tuple containing:
1551                 - number of boards that failed to build
1552                 - number of boards that issued warnings
1553         """
1554         self.commit_count = len(commits) if commits else 1
1555         self.commits = commits
1556         self._verbose = verbose
1557
1558         self.ResetResultSummary(board_selected)
1559         builderthread.Mkdir(self.base_dir, parents = True)
1560         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1561                 commits is not None)
1562         self._PrepareOutputSpace()
1563         Print('\rStarting build...', newline=False)
1564         self.SetupBuild(board_selected, commits)
1565         self.ProcessResult(None)
1566
1567         # Create jobs to build all commits for each board
1568         for brd in board_selected.values():
1569             job = builderthread.BuilderJob()
1570             job.board = brd
1571             job.commits = commits
1572             job.keep_outputs = keep_outputs
1573             job.step = self._step
1574             self.queue.put(job)
1575
1576         term = threading.Thread(target=self.queue.join)
1577         term.setDaemon(True)
1578         term.start()
1579         while term.isAlive():
1580             term.join(100)
1581
1582         # Wait until we have processed all output
1583         self.out_queue.join()
1584         Print()
1585         self.ClearLine(0)
1586         return (self.fail, self.warned)