Merge branch 'master' of git://git.denx.de/u-boot-i2c

[oweals/u-boot.git] / tools / buildman / builder.py

diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py
index 324239a84126bfc676c7f67fac37d60e252c9309..a5a2ffdfdf2c76d86fb78d765d9977245da652ac 100644 (file)
--- a/tools/buildman/builder.py
+++ b/tools/buildman/builder.py
@@ -1,9 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0+

 # Copyright (c) 2013 The Chromium OS Authors.
 #
 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
 #
 # Copyright (c) 2013 The Chromium OS Authors.
 #
 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
 #

-# SPDX-License-Identifier:     GPL-2.0+
-#

 
 import collections
 from datetime import datetime, timedelta
 
 import collections
 from datetime import datetime, timedelta


@@ -12,14 +11,17 @@ import os
 import re
 import Queue
 import shutil
 import re
 import Queue
 import shutil

+import signal

 import string
 import sys
 import string
 import sys

+import threading

 import time
 
 import builderthread
 import command
 import gitutil
 import terminal
 import time
 
 import builderthread
 import command
 import gitutil
 import terminal

+from terminal import Print

 import toolchain
 
 
 import toolchain
 
 


@@ -92,15 +94,52 @@ u-boot/             source directory
 # Possible build outcomes
 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
 
 # Possible build outcomes
 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
 

-# Translate a commit subject into a valid filename
-trans_valid_chars = string.maketrans("/: ", "---")
-
+# Translate a commit subject into a valid filename (and handle unicode)
+trans_valid_chars = string.maketrans('/: ', '---')
+trans_valid_chars = trans_valid_chars.decode('latin-1')
+
+BASE_CONFIG_FILENAMES = [
+    'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
+]
+
+EXTRA_CONFIG_FILENAMES = [
+    '.config', '.config-spl', '.config-tpl',
+    'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
+    'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
+]
+
+class Config:
+    """Holds information about configuration settings for a board."""
+    def __init__(self, config_filename, target):
+        self.target = target
+        self.config = {}
+        for fname in config_filename:
+            self.config[fname] = {}
+
+    def Add(self, fname, key, value):
+        self.config[fname][key] = value
+
+    def __hash__(self):
+        val = 0
+        for fname in self.config:
+            for key, value in self.config[fname].iteritems():
+                print key, value
+                val = val ^ hash(key) & hash(value)
+        return val
+
+class Environment:
+    """Holds information about environment variables for a board."""
+    def __init__(self, target):
+        self.target = target
+        self.environment = {}
+
+    def Add(self, key, value):
+        self.environment[key] = value

 
 class Builder:
     """Class for building U-Boot for a particular commit.
 
     Public members: (many should ->private)
 
 class Builder:
     """Class for building U-Boot for a particular commit.
 
     Public members: (many should ->private)

-        active: True if the builder is active and has not been stopped

         already_done: Number of builds already completed
         base_dir: Base directory to use for builder
         checkout: True to check out source, False to skip that step.
         already_done: Number of builds already completed
         base_dir: Base directory to use for builder
         checkout: True to check out source, False to skip that step.


@@ -165,15 +204,28 @@ class Builder:
                     value is itself a dictionary:
                         key: function name
                         value: Size of function in bytes
                     value is itself a dictionary:
                         key: function name
                         value: Size of function in bytes

+            config: Dictionary keyed by filename - e.g. '.config'. Each
+                    value is itself a dictionary:
+                        key: config name
+                        value: config value
+            environment: Dictionary keyed by environment variable, Each
+                     value is the value of environment variable.

         """
         """

-        def __init__(self, rc, err_lines, sizes, func_sizes):
+        def __init__(self, rc, err_lines, sizes, func_sizes, config,
+                     environment):

             self.rc = rc
             self.err_lines = err_lines
             self.sizes = sizes
             self.func_sizes = func_sizes
             self.rc = rc
             self.err_lines = err_lines
             self.sizes = sizes
             self.func_sizes = func_sizes

+            self.config = config
+            self.environment = environment

 
     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 
     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,

-                 gnu_make='make', checkout=True, show_unknown=True, step=1):
+                 gnu_make='make', checkout=True, show_unknown=True, step=1,
+                 no_subdirs=False, full_path=False, verbose_build=False,
+                 incremental=False, per_board_out_dir=False,
+                 config_only=False, squash_config_y=False,
+                 warnings_as_errors=False):

         """Create a new Builder object
 
         Args:
         """Create a new Builder object
 
         Args:


@@ -187,12 +239,23 @@ class Builder:
                 This is used for testing.
             show_unknown: Show unknown boards (those not built) in summary
             step: 1 to process every commit, n to process every nth commit
                 This is used for testing.
             show_unknown: Show unknown boards (those not built) in summary
             step: 1 to process every commit, n to process every nth commit

+            no_subdirs: Don't create subdirectories when building current
+                source for a single board
+            full_path: Return the full path in CROSS_COMPILE and don't set
+                PATH
+            verbose_build: Run build with V=1 and don't use 'make -s'
+            incremental: Always perform incremental builds; don't run make
+                mrproper when configuring
+            per_board_out_dir: Build in a separate persistent directory per
+                board rather than a thread-specific directory
+            config_only: Only configure each build, don't build it
+            squash_config_y: Convert CONFIG options with the value 'y' to '1'
+            warnings_as_errors: Treat all compiler warnings as errors

         """
         self.toolchains = toolchains
         self.base_dir = base_dir
         self._working_dir = os.path.join(base_dir, '.bm-work')
         self.threads = []
         """
         self.toolchains = toolchains
         self.base_dir = base_dir
         self._working_dir = os.path.join(base_dir, '.bm-work')
         self.threads = []

-        self.active = True

         self.do_make = self.Make
         self.gnu_make = gnu_make
         self.checkout = checkout
         self.do_make = self.Make
         self.gnu_make = gnu_make
         self.checkout = checkout


@@ -212,7 +275,16 @@ class Builder:
         self._step = step
         self.in_tree = False
         self._error_lines = 0
         self._step = step
         self.in_tree = False
         self._error_lines = 0

-
+        self.no_subdirs = no_subdirs
+        self.full_path = full_path
+        self.verbose_build = verbose_build
+        self.config_only = config_only
+        self.squash_config_y = squash_config_y
+        self.config_filenames = BASE_CONFIG_FILENAMES
+        if not self.squash_config_y:
+            self.config_filenames += EXTRA_CONFIG_FILENAMES
+
+        self.warnings_as_errors = warnings_as_errors

         self.col = terminal.Color()
 
         self._re_function = re.compile('(.*): In function.*')
         self.col = terminal.Color()
 
         self._re_function = re.compile('(.*): In function.*')


@@ -223,7 +295,8 @@ class Builder:
         self.queue = Queue.Queue()
         self.out_queue = Queue.Queue()
         for i in range(self.num_threads):
         self.queue = Queue.Queue()
         self.out_queue = Queue.Queue()
         for i in range(self.num_threads):

-            t = builderthread.BuilderThread(self, i)
+            t = builderthread.BuilderThread(self, i, incremental,
+                    per_board_out_dir)

             t.setDaemon(True)
             t.start()
             self.threads.append(t)
             t.setDaemon(True)
             t.start()
             self.threads.append(t)


@@ -237,14 +310,21 @@ class Builder:
         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
         self.re_make_err = re.compile('|'.join(ignore_lines))
 
         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
         self.re_make_err = re.compile('|'.join(ignore_lines))
 

+        # Handle existing graceful with SIGINT / Ctrl-C
+        signal.signal(signal.SIGINT, self.signal_handler)
+

     def __del__(self):
         """Get rid of all threads created by the builder"""
         for t in self.threads:
             del t
 
     def __del__(self):
         """Get rid of all threads created by the builder"""
         for t in self.threads:
             del t
 

+    def signal_handler(self, signal, frame):
+        sys.exit(1)
+

     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
                           show_detail=False, show_bloat=False,
     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
                           show_detail=False, show_bloat=False,

-                          list_error_boards=False):
+                          list_error_boards=False, show_config=False,
+                          show_environment=False):

         """Setup display options for the builder.
 
         show_errors: True to show summarised error/warning info
         """Setup display options for the builder.
 
         show_errors: True to show summarised error/warning info


@@ -252,12 +332,16 @@ class Builder:
         show_detail: Show detail for each board
         show_bloat: Show detail for each function
         list_error_boards: Show the boards which caused each error/warning
         show_detail: Show detail for each board
         show_bloat: Show detail for each function
         list_error_boards: Show the boards which caused each error/warning

+        show_config: Show config deltas
+        show_environment: Show environment deltas

         """
         self._show_errors = show_errors
         self._show_sizes = show_sizes
         self._show_detail = show_detail
         self._show_bloat = show_bloat
         self._list_error_boards = list_error_boards
         """
         self._show_errors = show_errors
         self._show_sizes = show_sizes
         self._show_detail = show_detail
         self._show_bloat = show_bloat
         self._list_error_boards = list_error_boards

+        self._show_config = show_config
+        self._show_environment = show_environment

 
     def _AddTimestamp(self):
         """Add a new timestamp to the list and record the build period.
 
     def _AddTimestamp(self):
         """Add a new timestamp to the list and record the build period.


@@ -299,8 +383,8 @@ class Builder:
             length: Length of new line, in characters
         """
         if length < self.last_line_len:
             length: Length of new line, in characters
         """
         if length < self.last_line_len:

-            print ' ' * (self.last_line_len - length),
-            print '\r',
+            Print(' ' * (self.last_line_len - length), newline=False)
+            Print('\r', newline=False)

         self.last_line_len = length
         sys.stdout.flush()
 
         self.last_line_len = length
         sys.stdout.flush()
 


@@ -325,6 +409,9 @@ class Builder:
         cmd = [self.gnu_make] + list(args)
         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
                 cwd=cwd, raise_on_error=False, **kwargs)
         cmd = [self.gnu_make] + list(args)
         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
                 cwd=cwd, raise_on_error=False, **kwargs)

+        if self.verbose_build:
+            result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
+            result.combined = '%s\n' % (' '.join(cmd)) + result.combined

         return result
 
     def ProcessResult(self, result):
         return result
 
     def ProcessResult(self, result):


@@ -338,11 +425,6 @@ class Builder:
         if result:
             target = result.brd.target
 
         if result:
             target = result.brd.target
 

-            if result.return_code < 0:
-                self.active = False
-                command.StopAll()
-                return
-

             self.upto += 1
             if result.return_code != 0:
                 self.fail += 1
             self.upto += 1
             if result.return_code != 0:
                 self.fail += 1


@@ -351,7 +433,7 @@ class Builder:
             if result.already_done:
                 self.already_done += 1
             if self._verbose:
             if result.already_done:
                 self.already_done += 1
             if self._verbose:

-                print '\r',
+                Print('\r', newline=False)

                 self.ClearLine(0)
                 boards_selected = {target : result.brd}
                 self.ResetResultSummary(boards_selected)
                 self.ClearLine(0)
                 boards_selected = {target : result.brd}
                 self.ResetResultSummary(boards_selected)


@@ -379,8 +461,8 @@ class Builder:
                     self.commit_count)
 
         name += target
                     self.commit_count)
 
         name += target

-        print line + name,
-        length = 14 + len(name)
+        Print(line + name, newline=False)
+        length = 16 + len(name)

         self.ClearLine(length)
 
     def _GetOutputDir(self, commit_upto):
         self.ClearLine(length)
 
     def _GetOutputDir(self, commit_upto):


@@ -391,15 +473,17 @@ class Builder:
         Args:
             commit_upto: Commit number to use (0..self.count-1)
         """
         Args:
             commit_upto: Commit number to use (0..self.count-1)
         """

+        commit_dir = None

         if self.commits:
             commit = self.commits[commit_upto]
             subject = commit.subject.translate(trans_valid_chars)
             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
                     self.commit_count, commit.hash, subject[:20]))
         if self.commits:
             commit = self.commits[commit_upto]
             subject = commit.subject.translate(trans_valid_chars)
             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
                     self.commit_count, commit.hash, subject[:20]))

-        else:
+        elif not self.no_subdirs:

             commit_dir = 'current'
             commit_dir = 'current'

-        output_dir = os.path.join(self.base_dir, commit_dir)
-        return output_dir
+        if not commit_dir:
+            return self.base_dir
+        return os.path.join(self.base_dir, commit_dir)

 
     def GetBuildDir(self, commit_upto, target):
         """Get the name of the build directory for a commit number
 
     def GetBuildDir(self, commit_upto, target):
         """Get the name of the build directory for a commit number


@@ -495,7 +579,7 @@ class Builder:
             try:
                 size, type, name = line[:-1].split()
             except:
             try:
                 size, type, name = line[:-1].split()
             except:

-                print "Invalid line in file '%s': '%s'" % (fname, line[:-1])
+                Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))

                 continue
             if type in 'tTdDbB':
                 # function names begin with '.' on 64-bit powerpc
                 continue
             if type in 'tTdDbB':
                 # function names begin with '.' on 64-bit powerpc


@@ -504,13 +588,78 @@ class Builder:
                 sym[name] = sym.get(name, 0) + int(size, 16)
         return sym
 
                 sym[name] = sym.get(name, 0) + int(size, 16)
         return sym
 

-    def GetBuildOutcome(self, commit_upto, target, read_func_sizes):
+    def _ProcessConfig(self, fname):
+        """Read in a .config, autoconf.mk or autoconf.h file
+
+        This function handles all config file types. It ignores comments and
+        any #defines which don't start with CONFIG_.
+
+        Args:
+            fname: Filename to read
+
+        Returns:
+            Dictionary:
+                key: Config name (e.g. CONFIG_DM)
+                value: Config value (e.g. 1)
+        """
+        config = {}
+        if os.path.exists(fname):
+            with open(fname) as fd:
+                for line in fd:
+                    line = line.strip()
+                    if line.startswith('#define'):
+                        values = line[8:].split(' ', 1)
+                        if len(values) > 1:
+                            key, value = values
+                        else:
+                            key = values[0]
+                            value = '1' if self.squash_config_y else ''
+                        if not key.startswith('CONFIG_'):
+                            continue
+                    elif not line or line[0] in ['#', '*', '/']:
+                        continue
+                    else:
+                        key, value = line.split('=', 1)
+                    if self.squash_config_y and value == 'y':
+                        value = '1'
+                    config[key] = value
+        return config
+
+    def _ProcessEnvironment(self, fname):
+        """Read in a uboot.env file
+
+        This function reads in environment variables from a file.
+
+        Args:
+            fname: Filename to read
+
+        Returns:
+            Dictionary:
+                key: environment variable (e.g. bootlimit)
+                value: value of environment variable (e.g. 1)
+        """
+        environment = {}
+        if os.path.exists(fname):
+            with open(fname) as fd:
+                for line in fd.read().split('\0'):
+                    try:
+                        key, value = line.split('=', 1)
+                        environment[key] = value
+                    except ValueError:
+                        # ignore lines we can't parse
+                        pass
+        return environment
+
+    def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
+                        read_config, read_environment):

         """Work out the outcome of a build.
 
         Args:
             commit_upto: Commit number to check (0..n-1)
             target: Target board to check
             read_func_sizes: True to read function size information
         """Work out the outcome of a build.
 
         Args:
             commit_upto: Commit number to check (0..n-1)
             target: Target board to check
             read_func_sizes: True to read function size information

+            read_config: True to read .config and autoconf.h files
+            read_environment: True to read uboot.env files

 
         Returns:
             Outcome object
 
         Returns:
             Outcome object


@@ -519,6 +668,8 @@ class Builder:
         sizes_file = self.GetSizesFile(commit_upto, target)
         sizes = {}
         func_sizes = {}
         sizes_file = self.GetSizesFile(commit_upto, target)
         sizes = {}
         func_sizes = {}

+        config = {}
+        environment = {}

         if os.path.exists(done_file):
             with open(done_file, 'r') as fd:
                 return_code = int(fd.readline())
         if os.path.exists(done_file):
             with open(done_file, 'r') as fd:
                 return_code = int(fd.readline())


@@ -562,17 +713,32 @@ class Builder:
                                                                     '')
                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 
                                                                     '')
                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 

-            return Builder.Outcome(rc, err_lines, sizes, func_sizes)
+            if read_config:
+                output_dir = self.GetBuildDir(commit_upto, target)
+                for name in self.config_filenames:
+                    fname = os.path.join(output_dir, name)
+                    config[name] = self._ProcessConfig(fname)

 
 

-        return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {})
+            if read_environment:
+                output_dir = self.GetBuildDir(commit_upto, target)
+                fname = os.path.join(output_dir, 'uboot.env')
+                environment = self._ProcessEnvironment(fname)

 
 

-    def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes):
+            return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
+                                   environment)
+
+        return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
+
+    def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
+                         read_config, read_environment):

         """Calculate a summary of the results of building a commit.
 
         Args:
             board_selected: Dict containing boards to summarise
             commit_upto: Commit number to summarize (0..self.count-1)
             read_func_sizes: True to read function size information
         """Calculate a summary of the results of building a commit.
 
         Args:
             board_selected: Dict containing boards to summarise
             commit_upto: Commit number to summarize (0..self.count-1)
             read_func_sizes: True to read function size information

+            read_config: True to read .config and autoconf.h files
+            read_environment: True to read uboot.env files

 
         Returns:
             Tuple:
 
         Returns:
             Tuple:


@@ -584,6 +750,14 @@ class Builder:
                 List containing a summary of warning lines
                 Dict keyed by error line, containing a list of the Board
                     objects with that warning
                 List containing a summary of warning lines
                 Dict keyed by error line, containing a list of the Board
                     objects with that warning

+                Dictionary keyed by board.target. Each value is a dictionary:
+                    key: filename - e.g. '.config'
+                    value is itself a dictionary:
+                        key: config name
+                        value: config value
+                Dictionary keyed by board.target. Each value is a dictionary:
+                    key: environment variable
+                    value: value of environment variable

         """
         def AddLine(lines_summary, lines_boards, line, board):
             line = line.rstrip()
         """
         def AddLine(lines_summary, lines_boards, line, board):
             line = line.rstrip()


@@ -598,10 +772,13 @@ class Builder:
         err_lines_boards = {}
         warn_lines_summary = []
         warn_lines_boards = {}
         err_lines_boards = {}
         warn_lines_summary = []
         warn_lines_boards = {}

+        config = {}
+        environment = {}

 
         for board in boards_selected.itervalues():
             outcome = self.GetBuildOutcome(commit_upto, board.target,
 
         for board in boards_selected.itervalues():
             outcome = self.GetBuildOutcome(commit_upto, board.target,

-                                           read_func_sizes)
+                                           read_func_sizes, read_config,
+                                           read_environment)

             board_dict[board.target] = outcome
             last_func = None
             last_was_warning = False
             board_dict[board.target] = outcome
             last_func = None
             last_was_warning = False


@@ -627,8 +804,21 @@ class Builder:
                                     line, board)
                         last_was_warning = is_warning
                         last_func = None
                                     line, board)
                         last_was_warning = is_warning
                         last_func = None

+            tconfig = Config(self.config_filenames, board.target)
+            for fname in self.config_filenames:
+                if outcome.config:
+                    for key, value in outcome.config[fname].iteritems():
+                        tconfig.Add(fname, key, value)
+            config[board.target] = tconfig
+
+            tenvironment = Environment(board.target)
+            if outcome.environment:
+                for key, value in outcome.environment.iteritems():
+                    tenvironment.Add(key, value)
+            environment[board.target] = tenvironment
+

         return (board_dict, err_lines_summary, err_lines_boards,
         return (board_dict, err_lines_summary, err_lines_boards,

-                warn_lines_summary, warn_lines_boards)
+                warn_lines_summary, warn_lines_boards, config, environment)

 
     def AddOutcome(self, board_dict, arch_list, changes, char, color):
         """Add an output to our list of outcomes for each architecture
 
     def AddOutcome(self, board_dict, arch_list, changes, char, color):
         """Add an output to our list of outcomes for each architecture


@@ -652,7 +842,7 @@ class Builder:
                 arch = 'unknown'
             str = self.col.Color(color, ' ' + target)
             if not arch in done_arch:
                 arch = 'unknown'
             str = self.col.Color(color, ' ' + target)
             if not arch in done_arch:

-                str = self.col.Color(color, char) + '  ' + str
+                str = ' %s  %s' % (self.col.Color(color, char), str)

                 done_arch[arch] = True
             if not arch in arch_list:
                 arch_list[arch] = str
                 done_arch[arch] = True
             if not arch in arch_list:
                 arch_list[arch] = str


@@ -681,11 +871,14 @@ class Builder:
         """
         self._base_board_dict = {}
         for board in board_selected:
         """
         self._base_board_dict = {}
         for board in board_selected:

-            self._base_board_dict[board] = Builder.Outcome(0, [], [], {})
+            self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
+                                                           {})

         self._base_err_lines = []
         self._base_warn_lines = []
         self._base_err_line_boards = {}
         self._base_warn_line_boards = {}
         self._base_err_lines = []
         self._base_warn_lines = []
         self._base_err_line_boards = {}
         self._base_warn_line_boards = {}

+        self._base_config = None
+        self._base_environment = None

 
     def PrintFuncSizeDetail(self, fname, old, new):
         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 
     def PrintFuncSizeDetail(self, fname, old, new):
         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0


@@ -719,20 +912,20 @@ class Builder:
         delta.reverse()
 
         args = [add, -remove, grow, -shrink, up, -down, up - down]
         delta.reverse()
 
         args = [add, -remove, grow, -shrink, up, -down, up - down]

-        if max(args) == 0:
+        if max(args) == 0 and min(args) == 0:

             return
         args = [self.ColourNum(x) for x in args]
         indent = ' ' * 15
             return
         args = [self.ColourNum(x) for x in args]
         indent = ' ' * 15

-        print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
-               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
-        print '%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
-                                        'delta')
+        Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
+              tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
+        Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
+                                         'delta'))

         for diff, name in delta:
             if diff:
                 color = self.col.RED if diff > 0 else self.col.GREEN
                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
                         old.get(name, '-'), new.get(name,'-'), diff)
         for diff, name in delta:
             if diff:
                 color = self.col.RED if diff > 0 else self.col.GREEN
                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
                         old.get(name, '-'), new.get(name,'-'), diff)

-                print self.col.Color(color, msg)
+                Print(msg, colour=color)

 
 
     def PrintSizeDetail(self, target_list, show_bloat):
 
 
     def PrintSizeDetail(self, target_list, show_bloat):


@@ -757,11 +950,12 @@ class Builder:
                     color = self.col.RED if diff > 0 else self.col.GREEN
                 msg = ' %s %+d' % (name, diff)
                 if not printed_target:
                     color = self.col.RED if diff > 0 else self.col.GREEN
                 msg = ' %s %+d' % (name, diff)
                 if not printed_target:

-                    print '%10s  %-15s:' % ('', result['_target']),
+                    Print('%10s  %-15s:' % ('', result['_target']),
+                          newline=False)

                     printed_target = True
                     printed_target = True

-                print self.col.Color(color, msg),
+                Print(msg, colour=color, newline=False)

             if printed_target:
             if printed_target:

-                print
+                Print()

                 if show_bloat:
                     target = result['_target']
                     outcome = result['_outcome']
                 if show_bloat:
                     target = result['_target']
                     outcome = result['_outcome']


@@ -866,20 +1060,21 @@ class Builder:
                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
                     msg = ' %s %+1.1f' % (name, avg_diff)
                     if not printed_arch:
                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
                     msg = ' %s %+1.1f' % (name, avg_diff)
                     if not printed_arch:

-                        print '%10s: (for %d/%d boards)' % (arch, count,
-                                arch_count[arch]),
+                        Print('%10s: (for %d/%d boards)' % (arch, count,
+                              arch_count[arch]), newline=False)

                         printed_arch = True
                         printed_arch = True

-                    print self.col.Color(color, msg),
+                    Print(msg, colour=color, newline=False)

 
             if printed_arch:
 
             if printed_arch:

-                print
+                Print()

                 if show_detail:
                     self.PrintSizeDetail(target_list, show_bloat)
 
 
     def PrintResultSummary(self, board_selected, board_dict, err_lines,
                            err_line_boards, warn_lines, warn_line_boards,
                 if show_detail:
                     self.PrintSizeDetail(target_list, show_bloat)
 
 
     def PrintResultSummary(self, board_selected, board_dict, err_lines,
                            err_line_boards, warn_lines, warn_line_boards,

-                           show_sizes, show_detail, show_bloat):
+                           config, environment, show_sizes, show_detail,
+                           show_bloat, show_config, show_environment):

         """Compare results with the base results and display delta.
 
         Only boards mentioned in board_selected will be considered. This
         """Compare results with the base results and display delta.
 
         Only boards mentioned in board_selected will be considered. This


@@ -900,9 +1095,17 @@ class Builder:
                 none, or we don't want to print errors
             warn_line_boards: Dict keyed by warning line, containing a list of
                 the Board objects with that warning
                 none, or we don't want to print errors
             warn_line_boards: Dict keyed by warning line, containing a list of
                 the Board objects with that warning

+            config: Dictionary keyed by filename - e.g. '.config'. Each
+                    value is itself a dictionary:
+                        key: config name
+                        value: config value
+            environment: Dictionary keyed by environment variable, Each
+                     value is the value of environment variable.

             show_sizes: Show image size deltas
             show_detail: Show detail for each board
             show_bloat: Show detail for each function
             show_sizes: Show image size deltas
             show_detail: Show detail for each board
             show_bloat: Show detail for each function

+            show_config: Show config changes
+            show_environment: Show environment changes

         """
         def _BoardList(line, line_boards):
             """Helper function to get a line of boards containing a line
         """
         def _BoardList(line, line_boards):
             """Helper function to get a line of boards containing a line


@@ -916,7 +1119,8 @@ class Builder:
             if self._list_error_boards:
                 names = []
                 for board in line_boards[line]:
             if self._list_error_boards:
                 names = []
                 for board in line_boards[line]:

-                    names.append(board.target)
+                    if not board.target in names:
+                        names.append(board.target)

                 names_str = '(%s) ' % ','.join(names)
             else:
                 names_str = ''
                 names_str = '(%s) ' % ','.join(names)
             else:
                 names_str = ''


@@ -936,6 +1140,60 @@ class Builder:
                             _BoardList(line, base_line_boards) + line)
             return better_lines, worse_lines
 
                             _BoardList(line, base_line_boards) + line)
             return better_lines, worse_lines
 

+        def _CalcConfig(delta, name, config):
+            """Calculate configuration changes
+
+            Args:
+                delta: Type of the delta, e.g. '+'
+                name: name of the file which changed (e.g. .config)
+                config: configuration change dictionary
+                    key: config name
+                    value: config value
+            Returns:
+                String containing the configuration changes which can be
+                    printed
+            """
+            out = ''
+            for key in sorted(config.keys()):
+                out += '%s=%s ' % (key, config[key])
+            return '%s %s: %s' % (delta, name, out)
+
+        def _AddConfig(lines, name, config_plus, config_minus, config_change):
+            """Add changes in configuration to a list
+
+            Args:
+                lines: list to add to
+                name: config file name
+                config_plus: configurations added, dictionary
+                    key: config name
+                    value: config value
+                config_minus: configurations removed, dictionary
+                    key: config name
+                    value: config value
+                config_change: configurations changed, dictionary
+                    key: config name
+                    value: config value
+            """
+            if config_plus:
+                lines.append(_CalcConfig('+', name, config_plus))
+            if config_minus:
+                lines.append(_CalcConfig('-', name, config_minus))
+            if config_change:
+                lines.append(_CalcConfig('c', name, config_change))
+
+        def _OutputConfigInfo(lines):
+            for line in lines:
+                if not line:
+                    continue
+                if line[0] == '+':
+                    col = self.col.GREEN
+                elif line[0] == '-':
+                    col = self.col.RED
+                elif line[0] == 'c':
+                    col = self.col.YELLOW
+                Print('   ' + line, newline=True, colour=col)
+
+

         better = []     # List of boards fixed since last commit
         worse = []      # List of new broken boards since last commit
         new = []        # List of boards that didn't exist last time
         better = []     # List of boards fixed since last commit
         worse = []      # List of new broken boards since last commit
         new = []        # List of boards that didn't exist last time


@@ -977,31 +1235,162 @@ class Builder:
                 self.AddOutcome(board_selected, arch_list, unknown, '?',
                         self.col.MAGENTA)
             for arch, target_list in arch_list.iteritems():
                 self.AddOutcome(board_selected, arch_list, unknown, '?',
                         self.col.MAGENTA)
             for arch, target_list in arch_list.iteritems():

-                print '%10s: %s' % (arch, target_list)
+                Print('%10s: %s' % (arch, target_list))

                 self._error_lines += 1
             if better_err:
                 self._error_lines += 1
             if better_err:

-                print self.col.Color(self.col.GREEN, '\n'.join(better_err))
+                Print('\n'.join(better_err), colour=self.col.GREEN)

                 self._error_lines += 1
             if worse_err:
                 self._error_lines += 1
             if worse_err:

-                print self.col.Color(self.col.RED, '\n'.join(worse_err))
+                Print('\n'.join(worse_err), colour=self.col.RED)

                 self._error_lines += 1
             if better_warn:
                 self._error_lines += 1
             if better_warn:

-                print self.col.Color(self.col.YELLOW, '\n'.join(better_warn))
+                Print('\n'.join(better_warn), colour=self.col.CYAN)

                 self._error_lines += 1
             if worse_warn:
                 self._error_lines += 1
             if worse_warn:

-                print self.col.Color(self.col.MAGENTA, '\n'.join(worse_warn))
+                Print('\n'.join(worse_warn), colour=self.col.MAGENTA)

                 self._error_lines += 1
 
         if show_sizes:
             self.PrintSizeSummary(board_selected, board_dict, show_detail,
                                   show_bloat)
 
                 self._error_lines += 1
 
         if show_sizes:
             self.PrintSizeSummary(board_selected, board_dict, show_detail,
                                   show_bloat)
 

+        if show_environment and self._base_environment:
+            lines = []
+
+            for target in board_dict:
+                if target not in board_selected:
+                    continue
+
+                tbase = self._base_environment[target]
+                tenvironment = environment[target]
+                environment_plus = {}
+                environment_minus = {}
+                environment_change = {}
+                base = tbase.environment
+                for key, value in tenvironment.environment.iteritems():
+                    if key not in base:
+                        environment_plus[key] = value
+                for key, value in base.iteritems():
+                    if key not in tenvironment.environment:
+                        environment_minus[key] = value
+                for key, value in base.iteritems():
+                    new_value = tenvironment.environment.get(key)
+                    if new_value and value != new_value:
+                        desc = '%s -> %s' % (value, new_value)
+                        environment_change[key] = desc
+
+                _AddConfig(lines, target, environment_plus, environment_minus,
+                           environment_change)
+
+            _OutputConfigInfo(lines)
+
+        if show_config and self._base_config:
+            summary = {}
+            arch_config_plus = {}
+            arch_config_minus = {}
+            arch_config_change = {}
+            arch_list = []
+
+            for target in board_dict:
+                if target not in board_selected:
+                    continue
+                arch = board_selected[target].arch
+                if arch not in arch_list:
+                    arch_list.append(arch)
+
+            for arch in arch_list:
+                arch_config_plus[arch] = {}
+                arch_config_minus[arch] = {}
+                arch_config_change[arch] = {}
+                for name in self.config_filenames:
+                    arch_config_plus[arch][name] = {}
+                    arch_config_minus[arch][name] = {}
+                    arch_config_change[arch][name] = {}
+
+            for target in board_dict:
+                if target not in board_selected:
+                    continue
+
+                arch = board_selected[target].arch
+
+                all_config_plus = {}
+                all_config_minus = {}
+                all_config_change = {}
+                tbase = self._base_config[target]
+                tconfig = config[target]
+                lines = []
+                for name in self.config_filenames:
+                    if not tconfig.config[name]:
+                        continue
+                    config_plus = {}
+                    config_minus = {}
+                    config_change = {}
+                    base = tbase.config[name]
+                    for key, value in tconfig.config[name].iteritems():
+                        if key not in base:
+                            config_plus[key] = value
+                            all_config_plus[key] = value
+                    for key, value in base.iteritems():
+                        if key not in tconfig.config[name]:
+                            config_minus[key] = value
+                            all_config_minus[key] = value
+                    for key, value in base.iteritems():
+                        new_value = tconfig.config.get(key)
+                        if new_value and value != new_value:
+                            desc = '%s -> %s' % (value, new_value)
+                            config_change[key] = desc
+                            all_config_change[key] = desc
+
+                    arch_config_plus[arch][name].update(config_plus)
+                    arch_config_minus[arch][name].update(config_minus)
+                    arch_config_change[arch][name].update(config_change)
+
+                    _AddConfig(lines, name, config_plus, config_minus,
+                               config_change)
+                _AddConfig(lines, 'all', all_config_plus, all_config_minus,
+                           all_config_change)
+                summary[target] = '\n'.join(lines)
+
+            lines_by_target = {}
+            for target, lines in summary.iteritems():
+                if lines in lines_by_target:
+                    lines_by_target[lines].append(target)
+                else:
+                    lines_by_target[lines] = [target]
+
+            for arch in arch_list:
+                lines = []
+                all_plus = {}
+                all_minus = {}
+                all_change = {}
+                for name in self.config_filenames:
+                    all_plus.update(arch_config_plus[arch][name])
+                    all_minus.update(arch_config_minus[arch][name])
+                    all_change.update(arch_config_change[arch][name])
+                    _AddConfig(lines, name, arch_config_plus[arch][name],
+                               arch_config_minus[arch][name],
+                               arch_config_change[arch][name])
+                _AddConfig(lines, 'all', all_plus, all_minus, all_change)
+                #arch_summary[target] = '\n'.join(lines)
+                if lines:
+                    Print('%s:' % arch)
+                    _OutputConfigInfo(lines)
+
+            for lines, targets in lines_by_target.iteritems():
+                if not lines:
+                    continue
+                Print('%s :' % ' '.join(sorted(targets)))
+                _OutputConfigInfo(lines.split('\n'))
+
+

         # Save our updated information for the next call to this function
         self._base_board_dict = board_dict
         self._base_err_lines = err_lines
         self._base_warn_lines = warn_lines
         self._base_err_line_boards = err_line_boards
         self._base_warn_line_boards = warn_line_boards
         # Save our updated information for the next call to this function
         self._base_board_dict = board_dict
         self._base_err_lines = err_lines
         self._base_warn_lines = warn_lines
         self._base_err_line_boards = err_line_boards
         self._base_warn_line_boards = warn_line_boards

+        self._base_config = config
+        self._base_environment = environment

 
         # Get a list of boards that did not get built, if needed
         not_built = []
 
         # Get a list of boards that did not get built, if needed
         not_built = []


@@ -1009,22 +1398,25 @@ class Builder:
             if not board in board_dict:
                 not_built.append(board)
         if not_built:
             if not board in board_dict:
                 not_built.append(board)
         if not_built:

-            print "Boards not built (%d): %s" % (len(not_built),
-                    ', '.join(not_built))
+            Print("Boards not built (%d): %s" % (len(not_built),
+                  ', '.join(not_built)))

 
     def ProduceResultSummary(self, commit_upto, commits, board_selected):
             (board_dict, err_lines, err_line_boards, warn_lines,
 
     def ProduceResultSummary(self, commit_upto, commits, board_selected):
             (board_dict, err_lines, err_line_boards, warn_lines,

-                    warn_line_boards) = self.GetResultSummary(
+             warn_line_boards, config, environment) = self.GetResultSummary(

                     board_selected, commit_upto,
                     board_selected, commit_upto,

-                    read_func_sizes=self._show_bloat)
+                    read_func_sizes=self._show_bloat,
+                    read_config=self._show_config,
+                    read_environment=self._show_environment)

             if commits:
                 msg = '%02d: %s' % (commit_upto + 1,
                         commits[commit_upto].subject)
             if commits:
                 msg = '%02d: %s' % (commit_upto + 1,
                         commits[commit_upto].subject)

-                print self.col.Color(self.col.BLUE, msg)
+                Print(msg, colour=self.col.BLUE)

             self.PrintResultSummary(board_selected, board_dict,
                     err_lines if self._show_errors else [], err_line_boards,
                     warn_lines if self._show_errors else [], warn_line_boards,
             self.PrintResultSummary(board_selected, board_dict,
                     err_lines if self._show_errors else [], err_line_boards,
                     warn_lines if self._show_errors else [], warn_line_boards,

-                    self._show_sizes, self._show_detail, self._show_bloat)
+                    config, environment, self._show_sizes, self._show_detail,
+                    self._show_bloat, self._show_config, self._show_environment)

 
     def ShowSummary(self, commits, board_selected):
         """Show a build summary for U-Boot for a given board list.
 
     def ShowSummary(self, commits, board_selected):
         """Show a build summary for U-Boot for a given board list.


@@ -1044,7 +1436,7 @@ class Builder:
         for commit_upto in range(0, self.commit_count, self._step):
             self.ProduceResultSummary(commit_upto, commits, board_selected)
         if not self._error_lines:
         for commit_upto in range(0, self.commit_count, self._step):
             self.ProduceResultSummary(commit_upto, commits, board_selected)
         if not self._error_lines:

-            print self.col.Color(self.col.GREEN, '(no errors to report)')
+            Print('(no errors to report)', colour=self.col.GREEN)

 
 
     def SetupBuild(self, board_selected, commits):
 
 
     def SetupBuild(self, board_selected, commits):


@@ -1089,8 +1481,10 @@ class Builder:
             if os.path.exists(git_dir):
                 gitutil.Fetch(git_dir, thread_dir)
             else:
             if os.path.exists(git_dir):
                 gitutil.Fetch(git_dir, thread_dir)
             else:

-                print 'Cloning repo for thread %d' % thread_num
+                Print('\rCloning repo for thread %d' % thread_num,
+                      newline=False)

                 gitutil.Clone(src_dir, thread_dir)
                 gitutil.Clone(src_dir, thread_dir)

+                Print('\r%s\r' % (' ' * 30), newline=False)

 
     def _PrepareWorkingSpace(self, max_threads, setup_git):
         """Prepare the working directory for use.
 
     def _PrepareWorkingSpace(self, max_threads, setup_git):
         """Prepare the working directory for use.


@@ -1112,12 +1506,20 @@ class Builder:
         create. Having left over directories is confusing when the user wants
         to check the output manually.
         """
         create. Having left over directories is confusing when the user wants
         to check the output manually.
         """

+        if not self.commits:
+            return

         dir_list = []
         for commit_upto in range(self.commit_count):
             dir_list.append(self._GetOutputDir(commit_upto))
 
         dir_list = []
         for commit_upto in range(self.commit_count):
             dir_list.append(self._GetOutputDir(commit_upto))
 

+        to_remove = []

         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
             if dirname not in dir_list:
         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
             if dirname not in dir_list:

+                to_remove.append(dirname)
+        if to_remove:
+            Print('Removing %d old build directories' % len(to_remove),
+                  newline=False)
+            for dirname in to_remove:

                 shutil.rmtree(dirname)
 
     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
                 shutil.rmtree(dirname)
 
     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):


@@ -1139,10 +1541,11 @@ class Builder:
         self._verbose = verbose
 
         self.ResetResultSummary(board_selected)
         self._verbose = verbose
 
         self.ResetResultSummary(board_selected)

-        builderthread.Mkdir(self.base_dir)
+        builderthread.Mkdir(self.base_dir, parents = True)

         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
                 commits is not None)
         self._PrepareOutputSpace()
         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
                 commits is not None)
         self._PrepareOutputSpace()

+        Print('\rStarting build...', newline=False)

         self.SetupBuild(board_selected, commits)
         self.ProcessResult(None)
 
         self.SetupBuild(board_selected, commits)
         self.ProcessResult(None)
 


@@ -1155,11 +1558,14 @@ class Builder:
             job.step = self._step
             self.queue.put(job)
 
             job.step = self._step
             self.queue.put(job)
 

-        # Wait until all jobs are started
-        self.queue.join()
+        term = threading.Thread(target=self.queue.join)
+        term.setDaemon(True)
+        term.start()
+        while term.isAlive():
+            term.join(100)

 
         # Wait until we have processed all output
         self.out_queue.join()
 
         # Wait until we have processed all output
         self.out_queue.join()

-        print
+        Print()

         self.ClearLine(0)
         return (self.fail, self.warned)
         self.ClearLine(0)
         return (self.fail, self.warned)