1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2013 The Chromium OS Authors.
4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
8 from datetime import datetime, timedelta
24 from terminal import Print
30 Please see README for user documentation, and you should be familiar with
31 that before trying to make sense of this.
33 Buildman works by keeping the machine as busy as possible, building different
34 commits for different boards on multiple CPUs at once.
36 The source repo (self.git_dir) contains all the commits to be built. Each
37 thread works on a single board at a time. It checks out the first commit,
38 configures it for that board, then builds it. Then it checks out the next
39 commit and builds it (typically without re-configuring). When it runs out
40 of commits, it gets another job from the builder and starts again with that
43 Clearly the builder threads could work either way - they could check out a
44 commit and then built it for all boards. Using separate directories for each
45 commit/board pair they could leave their build product around afterwards
48 The intent behind building a single board for multiple commits, is to make
49 use of incremental builds. Since each commit is built incrementally from
50 the previous one, builds are faster. Reconfiguring for a different board
51 removes all intermediate object files.
53 Many threads can be working at once, but each has its own working directory.
54 When a thread finishes a build, it puts the output files into a result
57 The base directory used by buildman is normally '../<branch>', i.e.
58 a directory higher than the source repository and named after the branch
61 Within the base directory, we have one subdirectory for each commit. Within
62 that is one subdirectory for each board. Within that is the build output for
63 that commit/board combination.
65 Buildman also create working directories for each thread, in a .bm-work/
66 subdirectory in the base dir.
68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
69 'seaboard', and say that us-net has two commits. We will have directories
72 us-net/ base directory
73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
78 02_of_02_g4ed4ebc_net--Check-tftp-comp/
84 00/ working directory for thread 0 (contains source checkout)
86 01/ working directory for thread 1
89 u-boot/ source directory
93 """Holds information about a particular error line we are outputing
95 char: Character representation: '+': error, '-': fixed error, 'w+': warning,
97 boards: List of Board objects which have line in the error/warning output
98 errline: The text of the error line
100 ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
102 # Possible build outcomes
103 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
105 # Translate a commit subject into a valid filename (and handle unicode)
106 trans_valid_chars = str.maketrans('/: ', '---')
108 BASE_CONFIG_FILENAMES = [
109 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
112 EXTRA_CONFIG_FILENAMES = [
113 '.config', '.config-spl', '.config-tpl',
114 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
115 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
119 """Holds information about configuration settings for a board."""
120 def __init__(self, config_filename, target):
123 for fname in config_filename:
124 self.config[fname] = {}
126 def Add(self, fname, key, value):
127 self.config[fname][key] = value
131 for fname in self.config:
132 for key, value in self.config[fname].items():
134 val = val ^ hash(key) & hash(value)
138 """Holds information about environment variables for a board."""
139 def __init__(self, target):
141 self.environment = {}
143 def Add(self, key, value):
144 self.environment[key] = value
147 """Class for building U-Boot for a particular commit.
149 Public members: (many should ->private)
150 already_done: Number of builds already completed
151 base_dir: Base directory to use for builder
152 checkout: True to check out source, False to skip that step.
153 This is used for testing.
154 col: terminal.Color() object
155 count: Number of commits to build
156 do_make: Method to call to invoke Make
157 fail: Number of builds that failed due to error
158 force_build: Force building even if a build already exists
159 force_config_on_failure: If a commit fails for a board, disable
160 incremental building for the next commit we build for that
161 board, so that we will see all warnings/errors again.
162 force_build_failures: If a previously-built build (i.e. built on
163 a previous run of buildman) is marked as failed, rebuild it.
164 git_dir: Git directory containing source repository
165 num_jobs: Number of jobs to run at once (passed to make as -j)
166 num_threads: Number of builder threads to run
167 out_queue: Queue of results to process
168 re_make_err: Compiled regular expression for ignore_lines
169 queue: Queue of jobs to run
170 threads: List of active threads
171 toolchains: Toolchains object to use for building
172 upto: Current commit number we are building (0.count-1)
173 warned: Number of builds that produced at least one warning
174 force_reconfig: Reconfigure U-Boot on each comiit. This disables
175 incremental building, where buildman reconfigures on the first
176 commit for a baord, and then just does an incremental build for
177 the following commits. In fact buildman will reconfigure and
178 retry for any failing commits, so generally the only effect of
179 this option is to slow things down.
180 in_tree: Build U-Boot in-tree instead of specifying an output
181 directory separate from the source code. This option is really
182 only useful for testing in-tree builds.
183 work_in_output: Use the output directory as the work directory and
184 don't write to a separate output directory.
187 _base_board_dict: Last-summarised Dict of boards
188 _base_err_lines: Last-summarised list of errors
189 _base_warn_lines: Last-summarised list of warnings
190 _build_period_us: Time taken for a single build (float object).
191 _complete_delay: Expected delay until completion (timedelta)
192 _next_delay_update: Next time we plan to display a progress update
194 _show_unknown: Show unknown boards (those not built) in summary
195 _timestamps: List of timestamps for the completion of the last
196 last _timestamp_count builds. Each is a datetime object.
197 _timestamp_count: Number of timestamps to keep in our list.
198 _working_dir: Base working directory containing all threads
201 """Records a build outcome for a single make invocation
204 rc: Outcome value (OUTCOME_...)
205 err_lines: List of error lines or [] if none
206 sizes: Dictionary of image size information, keyed by filename
207 - Each value is itself a dictionary containing
208 values for 'text', 'data' and 'bss', being the integer
209 size in bytes of each section.
210 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
211 value is itself a dictionary:
213 value: Size of function in bytes
214 config: Dictionary keyed by filename - e.g. '.config'. Each
215 value is itself a dictionary:
218 environment: Dictionary keyed by environment variable, Each
219 value is the value of environment variable.
221 def __init__(self, rc, err_lines, sizes, func_sizes, config,
224 self.err_lines = err_lines
226 self.func_sizes = func_sizes
228 self.environment = environment
230 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
231 gnu_make='make', checkout=True, show_unknown=True, step=1,
232 no_subdirs=False, full_path=False, verbose_build=False,
233 incremental=False, per_board_out_dir=False,
234 config_only=False, squash_config_y=False,
235 warnings_as_errors=False, work_in_output=False):
236 """Create a new Builder object
239 toolchains: Toolchains object to use for building
240 base_dir: Base directory to use for builder
241 git_dir: Git directory containing source repository
242 num_threads: Number of builder threads to run
243 num_jobs: Number of jobs to run at once (passed to make as -j)
244 gnu_make: the command name of GNU Make.
245 checkout: True to check out source, False to skip that step.
246 This is used for testing.
247 show_unknown: Show unknown boards (those not built) in summary
248 step: 1 to process every commit, n to process every nth commit
249 no_subdirs: Don't create subdirectories when building current
250 source for a single board
251 full_path: Return the full path in CROSS_COMPILE and don't set
253 verbose_build: Run build with V=1 and don't use 'make -s'
254 incremental: Always perform incremental builds; don't run make
255 mrproper when configuring
256 per_board_out_dir: Build in a separate persistent directory per
257 board rather than a thread-specific directory
258 config_only: Only configure each build, don't build it
259 squash_config_y: Convert CONFIG options with the value 'y' to '1'
260 warnings_as_errors: Treat all compiler warnings as errors
261 work_in_output: Use the output directory as the work directory and
262 don't write to a separate output directory.
264 self.toolchains = toolchains
265 self.base_dir = base_dir
267 self._working_dir = base_dir
269 self._working_dir = os.path.join(base_dir, '.bm-work')
271 self.do_make = self.Make
272 self.gnu_make = gnu_make
273 self.checkout = checkout
274 self.num_threads = num_threads
275 self.num_jobs = num_jobs
276 self.already_done = 0
277 self.force_build = False
278 self.git_dir = git_dir
279 self._show_unknown = show_unknown
280 self._timestamp_count = 10
281 self._build_period_us = None
282 self._complete_delay = None
283 self._next_delay_update = datetime.now()
284 self.force_config_on_failure = True
285 self.force_build_failures = False
286 self.force_reconfig = False
289 self._error_lines = 0
290 self.no_subdirs = no_subdirs
291 self.full_path = full_path
292 self.verbose_build = verbose_build
293 self.config_only = config_only
294 self.squash_config_y = squash_config_y
295 self.config_filenames = BASE_CONFIG_FILENAMES
296 self.work_in_output = work_in_output
297 if not self.squash_config_y:
298 self.config_filenames += EXTRA_CONFIG_FILENAMES
300 self.warnings_as_errors = warnings_as_errors
301 self.col = terminal.Color()
303 self._re_function = re.compile('(.*): In function.*')
304 self._re_files = re.compile('In file included from.*')
305 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
306 self._re_dtb_warning = re.compile('(.*): Warning .*')
307 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
309 self.queue = queue.Queue()
310 self.out_queue = queue.Queue()
311 for i in range(self.num_threads):
312 t = builderthread.BuilderThread(self, i, incremental,
316 self.threads.append(t)
318 t = builderthread.ResultThread(self)
321 self.threads.append(t)
323 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
324 self.re_make_err = re.compile('|'.join(ignore_lines))
326 # Handle existing graceful with SIGINT / Ctrl-C
327 signal.signal(signal.SIGINT, self.signal_handler)
330 """Get rid of all threads created by the builder"""
331 for t in self.threads:
334 def signal_handler(self, signal, frame):
337 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
338 show_detail=False, show_bloat=False,
339 list_error_boards=False, show_config=False,
340 show_environment=False):
341 """Setup display options for the builder.
343 show_errors: True to show summarised error/warning info
344 show_sizes: Show size deltas
345 show_detail: Show size delta detail for each board if show_sizes
346 show_bloat: Show detail for each function
347 list_error_boards: Show the boards which caused each error/warning
348 show_config: Show config deltas
349 show_environment: Show environment deltas
351 self._show_errors = show_errors
352 self._show_sizes = show_sizes
353 self._show_detail = show_detail
354 self._show_bloat = show_bloat
355 self._list_error_boards = list_error_boards
356 self._show_config = show_config
357 self._show_environment = show_environment
359 def _AddTimestamp(self):
360 """Add a new timestamp to the list and record the build period.
362 The build period is the length of time taken to perform a single
363 build (one board, one commit).
366 self._timestamps.append(now)
367 count = len(self._timestamps)
368 delta = self._timestamps[-1] - self._timestamps[0]
369 seconds = delta.total_seconds()
371 # If we have enough data, estimate build period (time taken for a
372 # single build) and therefore completion time.
373 if count > 1 and self._next_delay_update < now:
374 self._next_delay_update = now + timedelta(seconds=2)
376 self._build_period = float(seconds) / count
377 todo = self.count - self.upto
378 self._complete_delay = timedelta(microseconds=
379 self._build_period * todo * 1000000)
381 self._complete_delay -= timedelta(
382 microseconds=self._complete_delay.microseconds)
385 self._timestamps.popleft()
388 def SelectCommit(self, commit, checkout=True):
389 """Checkout the selected commit for this build
392 if checkout and self.checkout:
393 gitutil.Checkout(commit.hash)
395 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
399 commit: Commit object that is being built
400 brd: Board object that is being built
401 stage: Stage that we are at (mrproper, config, build)
402 cwd: Directory where make should be run
403 args: Arguments to pass to make
404 kwargs: Arguments to pass to command.RunPipe()
406 cmd = [self.gnu_make] + list(args)
407 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
408 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
409 if self.verbose_build:
410 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
411 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
414 def ProcessResult(self, result):
415 """Process the result of a build, showing progress information
418 result: A CommandResult object, which indicates the result for
421 col = terminal.Color()
423 target = result.brd.target
426 if result.return_code != 0:
430 if result.already_done:
431 self.already_done += 1
433 terminal.PrintClear()
434 boards_selected = {target : result.brd}
435 self.ResetResultSummary(boards_selected)
436 self.ProduceResultSummary(result.commit_upto, self.commits,
439 target = '(starting)'
441 # Display separate counts for ok, warned and fail
442 ok = self.upto - self.warned - self.fail
443 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
444 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
445 line += self.col.Color(self.col.RED, '%5d' % self.fail)
447 name = ' /%-5d ' % self.count
449 # Add our current completion time estimate
451 if self._complete_delay:
452 name += '%s : ' % self._complete_delay
453 # When building all boards for a commit, we can print a commit
455 if result and result.commit_upto is None:
456 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
460 terminal.PrintClear()
461 Print(line + name, newline=False)
463 def _GetOutputDir(self, commit_upto):
464 """Get the name of the output directory for a commit number
466 The output directory is typically .../<branch>/<commit>.
469 commit_upto: Commit number to use (0..self.count-1)
473 commit = self.commits[commit_upto]
474 subject = commit.subject.translate(trans_valid_chars)
475 # See _GetOutputSpaceRemovals() which parses this name
476 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
477 self.commit_count, commit.hash, subject[:20]))
478 elif not self.no_subdirs:
479 commit_dir = 'current'
482 return os.path.join(self.base_dir, commit_dir)
484 def GetBuildDir(self, commit_upto, target):
485 """Get the name of the build directory for a commit number
487 The build directory is typically .../<branch>/<commit>/<target>.
490 commit_upto: Commit number to use (0..self.count-1)
493 output_dir = self._GetOutputDir(commit_upto)
494 return os.path.join(output_dir, target)
496 def GetDoneFile(self, commit_upto, target):
497 """Get the name of the done file for a commit number
500 commit_upto: Commit number to use (0..self.count-1)
503 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
505 def GetSizesFile(self, commit_upto, target):
506 """Get the name of the sizes file for a commit number
509 commit_upto: Commit number to use (0..self.count-1)
512 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
514 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
515 """Get the name of the funcsizes file for a commit number and ELF file
518 commit_upto: Commit number to use (0..self.count-1)
520 elf_fname: Filename of elf image
522 return os.path.join(self.GetBuildDir(commit_upto, target),
523 '%s.sizes' % elf_fname.replace('/', '-'))
525 def GetObjdumpFile(self, commit_upto, target, elf_fname):
526 """Get the name of the objdump file for a commit number and ELF file
529 commit_upto: Commit number to use (0..self.count-1)
531 elf_fname: Filename of elf image
533 return os.path.join(self.GetBuildDir(commit_upto, target),
534 '%s.objdump' % elf_fname.replace('/', '-'))
536 def GetErrFile(self, commit_upto, target):
537 """Get the name of the err file for a commit number
540 commit_upto: Commit number to use (0..self.count-1)
543 output_dir = self.GetBuildDir(commit_upto, target)
544 return os.path.join(output_dir, 'err')
546 def FilterErrors(self, lines):
547 """Filter out errors in which we have no interest
549 We should probably use map().
552 lines: List of error lines, each a string
554 New list with only interesting lines included
558 if not self.re_make_err.search(line):
559 out_lines.append(line)
562 def ReadFuncSizes(self, fname, fd):
563 """Read function sizes from the output of 'nm'
566 fd: File containing data to read
567 fname: Filename we are reading from (just for errors)
570 Dictionary containing size of each function in bytes, indexed by
574 for line in fd.readlines():
577 size, type, name = line[:-1].split()
579 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
582 # function names begin with '.' on 64-bit powerpc
584 name = 'static.' + name.split('.')[0]
585 sym[name] = sym.get(name, 0) + int(size, 16)
588 def _ProcessConfig(self, fname):
589 """Read in a .config, autoconf.mk or autoconf.h file
591 This function handles all config file types. It ignores comments and
592 any #defines which don't start with CONFIG_.
595 fname: Filename to read
599 key: Config name (e.g. CONFIG_DM)
600 value: Config value (e.g. 1)
603 if os.path.exists(fname):
604 with open(fname) as fd:
607 if line.startswith('#define'):
608 values = line[8:].split(' ', 1)
613 value = '1' if self.squash_config_y else ''
614 if not key.startswith('CONFIG_'):
616 elif not line or line[0] in ['#', '*', '/']:
619 key, value = line.split('=', 1)
620 if self.squash_config_y and value == 'y':
625 def _ProcessEnvironment(self, fname):
626 """Read in a uboot.env file
628 This function reads in environment variables from a file.
631 fname: Filename to read
635 key: environment variable (e.g. bootlimit)
636 value: value of environment variable (e.g. 1)
639 if os.path.exists(fname):
640 with open(fname) as fd:
641 for line in fd.read().split('\0'):
643 key, value = line.split('=', 1)
644 environment[key] = value
646 # ignore lines we can't parse
650 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
651 read_config, read_environment):
652 """Work out the outcome of a build.
655 commit_upto: Commit number to check (0..n-1)
656 target: Target board to check
657 read_func_sizes: True to read function size information
658 read_config: True to read .config and autoconf.h files
659 read_environment: True to read uboot.env files
664 done_file = self.GetDoneFile(commit_upto, target)
665 sizes_file = self.GetSizesFile(commit_upto, target)
670 if os.path.exists(done_file):
671 with open(done_file, 'r') as fd:
673 return_code = int(fd.readline())
675 # The file may be empty due to running out of disk space.
679 err_file = self.GetErrFile(commit_upto, target)
680 if os.path.exists(err_file):
681 with open(err_file, 'r') as fd:
682 err_lines = self.FilterErrors(fd.readlines())
684 # Decide whether the build was ok, failed or created warnings
692 # Convert size information to our simple format
693 if os.path.exists(sizes_file):
694 with open(sizes_file, 'r') as fd:
695 for line in fd.readlines():
696 values = line.split()
699 rodata = int(values[6], 16)
701 'all' : int(values[0]) + int(values[1]) +
703 'text' : int(values[0]) - rodata,
704 'data' : int(values[1]),
705 'bss' : int(values[2]),
708 sizes[values[5]] = size_dict
711 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
712 for fname in glob.glob(pattern):
713 with open(fname, 'r') as fd:
714 dict_name = os.path.basename(fname).replace('.sizes',
716 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
719 output_dir = self.GetBuildDir(commit_upto, target)
720 for name in self.config_filenames:
721 fname = os.path.join(output_dir, name)
722 config[name] = self._ProcessConfig(fname)
725 output_dir = self.GetBuildDir(commit_upto, target)
726 fname = os.path.join(output_dir, 'uboot.env')
727 environment = self._ProcessEnvironment(fname)
729 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
732 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
734 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
735 read_config, read_environment):
736 """Calculate a summary of the results of building a commit.
739 board_selected: Dict containing boards to summarise
740 commit_upto: Commit number to summarize (0..self.count-1)
741 read_func_sizes: True to read function size information
742 read_config: True to read .config and autoconf.h files
743 read_environment: True to read uboot.env files
747 Dict containing boards which passed building this commit.
748 keyed by board.target
749 List containing a summary of error lines
750 Dict keyed by error line, containing a list of the Board
751 objects with that error
752 List containing a summary of warning lines
753 Dict keyed by error line, containing a list of the Board
754 objects with that warning
755 Dictionary keyed by board.target. Each value is a dictionary:
756 key: filename - e.g. '.config'
757 value is itself a dictionary:
760 Dictionary keyed by board.target. Each value is a dictionary:
761 key: environment variable
762 value: value of environment variable
764 def AddLine(lines_summary, lines_boards, line, board):
766 if line in lines_boards:
767 lines_boards[line].append(board)
769 lines_boards[line] = [board]
770 lines_summary.append(line)
773 err_lines_summary = []
774 err_lines_boards = {}
775 warn_lines_summary = []
776 warn_lines_boards = {}
780 for board in boards_selected.values():
781 outcome = self.GetBuildOutcome(commit_upto, board.target,
782 read_func_sizes, read_config,
784 board_dict[board.target] = outcome
786 last_was_warning = False
787 for line in outcome.err_lines:
789 if (self._re_function.match(line) or
790 self._re_files.match(line)):
793 is_warning = (self._re_warning.match(line) or
794 self._re_dtb_warning.match(line))
795 is_note = self._re_note.match(line)
796 if is_warning or (last_was_warning and is_note):
798 AddLine(warn_lines_summary, warn_lines_boards,
800 AddLine(warn_lines_summary, warn_lines_boards,
804 AddLine(err_lines_summary, err_lines_boards,
806 AddLine(err_lines_summary, err_lines_boards,
808 last_was_warning = is_warning
810 tconfig = Config(self.config_filenames, board.target)
811 for fname in self.config_filenames:
813 for key, value in outcome.config[fname].items():
814 tconfig.Add(fname, key, value)
815 config[board.target] = tconfig
817 tenvironment = Environment(board.target)
818 if outcome.environment:
819 for key, value in outcome.environment.items():
820 tenvironment.Add(key, value)
821 environment[board.target] = tenvironment
823 return (board_dict, err_lines_summary, err_lines_boards,
824 warn_lines_summary, warn_lines_boards, config, environment)
826 def AddOutcome(self, board_dict, arch_list, changes, char, color):
827 """Add an output to our list of outcomes for each architecture
829 This simple function adds failing boards (changes) to the
830 relevant architecture string, so we can print the results out
831 sorted by architecture.
834 board_dict: Dict containing all boards
835 arch_list: Dict keyed by arch name. Value is a string containing
836 a list of board names which failed for that arch.
837 changes: List of boards to add to arch_list
838 color: terminal.Colour object
841 for target in changes:
842 if target in board_dict:
843 arch = board_dict[target].arch
846 str = self.col.Color(color, ' ' + target)
847 if not arch in done_arch:
848 str = ' %s %s' % (self.col.Color(color, char), str)
849 done_arch[arch] = True
850 if not arch in arch_list:
851 arch_list[arch] = str
853 arch_list[arch] += str
856 def ColourNum(self, num):
857 color = self.col.RED if num > 0 else self.col.GREEN
860 return self.col.Color(color, str(num))
862 def ResetResultSummary(self, board_selected):
863 """Reset the results summary ready for use.
865 Set up the base board list to be all those selected, and set the
866 error lines to empty.
868 Following this, calls to PrintResultSummary() will use this
869 information to work out what has changed.
872 board_selected: Dict containing boards to summarise, keyed by
875 self._base_board_dict = {}
876 for board in board_selected:
877 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
879 self._base_err_lines = []
880 self._base_warn_lines = []
881 self._base_err_line_boards = {}
882 self._base_warn_line_boards = {}
883 self._base_config = None
884 self._base_environment = None
886 def PrintFuncSizeDetail(self, fname, old, new):
887 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
888 delta, common = [], {}
895 if name not in common:
898 delta.append([-old[name], name])
901 if name not in common:
904 delta.append([new[name], name])
907 diff = new.get(name, 0) - old.get(name, 0)
909 grow, up = grow + 1, up + diff
911 shrink, down = shrink + 1, down - diff
912 delta.append([diff, name])
917 args = [add, -remove, grow, -shrink, up, -down, up - down]
918 if max(args) == 0 and min(args) == 0:
920 args = [self.ColourNum(x) for x in args]
922 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
923 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
924 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
926 for diff, name in delta:
928 color = self.col.RED if diff > 0 else self.col.GREEN
929 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
930 old.get(name, '-'), new.get(name,'-'), diff)
931 Print(msg, colour=color)
934 def PrintSizeDetail(self, target_list, show_bloat):
935 """Show details size information for each board
938 target_list: List of targets, each a dict containing:
939 'target': Target name
940 'total_diff': Total difference in bytes across all areas
941 <part_name>: Difference for that part
942 show_bloat: Show detail for each function
944 targets_by_diff = sorted(target_list, reverse=True,
945 key=lambda x: x['_total_diff'])
946 for result in targets_by_diff:
947 printed_target = False
948 for name in sorted(result):
950 if name.startswith('_'):
953 color = self.col.RED if diff > 0 else self.col.GREEN
954 msg = ' %s %+d' % (name, diff)
955 if not printed_target:
956 Print('%10s %-15s:' % ('', result['_target']),
958 printed_target = True
959 Print(msg, colour=color, newline=False)
963 target = result['_target']
964 outcome = result['_outcome']
965 base_outcome = self._base_board_dict[target]
966 for fname in outcome.func_sizes:
967 self.PrintFuncSizeDetail(fname,
968 base_outcome.func_sizes[fname],
969 outcome.func_sizes[fname])
972 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
974 """Print a summary of image sizes broken down by section.
976 The summary takes the form of one line per architecture. The
977 line contains deltas for each of the sections (+ means the section
978 got bigger, - means smaller). The numbers are the average number
979 of bytes that a board in this section increased by.
982 powerpc: (622 boards) text -0.0
983 arm: (285 boards) text -0.0
984 nds32: (3 boards) text -8.0
987 board_selected: Dict containing boards to summarise, keyed by
989 board_dict: Dict containing boards for which we built this
990 commit, keyed by board.target. The value is an Outcome object.
991 show_detail: Show size delta detail for each board
992 show_bloat: Show detail for each function
997 # Calculate changes in size for different image parts
998 # The previous sizes are in Board.sizes, for each board
999 for target in board_dict:
1000 if target not in board_selected:
1002 base_sizes = self._base_board_dict[target].sizes
1003 outcome = board_dict[target]
1004 sizes = outcome.sizes
1006 # Loop through the list of images, creating a dict of size
1007 # changes for each image/part. We end up with something like
1008 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1009 # which means that U-Boot data increased by 5 bytes and SPL
1010 # text decreased by 4.
1011 err = {'_target' : target}
1013 if image in base_sizes:
1014 base_image = base_sizes[image]
1015 # Loop through the text, data, bss parts
1016 for part in sorted(sizes[image]):
1017 diff = sizes[image][part] - base_image[part]
1020 if image == 'u-boot':
1023 name = image + ':' + part
1025 arch = board_selected[target].arch
1026 if not arch in arch_count:
1027 arch_count[arch] = 1
1029 arch_count[arch] += 1
1031 pass # Only add to our list when we have some stats
1032 elif not arch in arch_list:
1033 arch_list[arch] = [err]
1035 arch_list[arch].append(err)
1037 # We now have a list of image size changes sorted by arch
1038 # Print out a summary of these
1039 for arch, target_list in arch_list.items():
1040 # Get total difference for each type
1042 for result in target_list:
1044 for name, diff in result.items():
1045 if name.startswith('_'):
1049 totals[name] += diff
1052 result['_total_diff'] = total
1053 result['_outcome'] = board_dict[result['_target']]
1055 count = len(target_list)
1056 printed_arch = False
1057 for name in sorted(totals):
1060 # Display the average difference in this name for this
1062 avg_diff = float(diff) / count
1063 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1064 msg = ' %s %+1.1f' % (name, avg_diff)
1065 if not printed_arch:
1066 Print('%10s: (for %d/%d boards)' % (arch, count,
1067 arch_count[arch]), newline=False)
1069 Print(msg, colour=color, newline=False)
1074 self.PrintSizeDetail(target_list, show_bloat)
1077 def PrintResultSummary(self, board_selected, board_dict, err_lines,
1078 err_line_boards, warn_lines, warn_line_boards,
1079 config, environment, show_sizes, show_detail,
1080 show_bloat, show_config, show_environment):
1081 """Compare results with the base results and display delta.
1083 Only boards mentioned in board_selected will be considered. This
1084 function is intended to be called repeatedly with the results of
1085 each commit. It therefore shows a 'diff' between what it saw in
1086 the last call and what it sees now.
1089 board_selected: Dict containing boards to summarise, keyed by
1091 board_dict: Dict containing boards for which we built this
1092 commit, keyed by board.target. The value is an Outcome object.
1093 err_lines: A list of errors for this commit, or [] if there is
1094 none, or we don't want to print errors
1095 err_line_boards: Dict keyed by error line, containing a list of
1096 the Board objects with that error
1097 warn_lines: A list of warnings for this commit, or [] if there is
1098 none, or we don't want to print errors
1099 warn_line_boards: Dict keyed by warning line, containing a list of
1100 the Board objects with that warning
1101 config: Dictionary keyed by filename - e.g. '.config'. Each
1102 value is itself a dictionary:
1105 environment: Dictionary keyed by environment variable, Each
1106 value is the value of environment variable.
1107 show_sizes: Show image size deltas
1108 show_detail: Show size delta detail for each board if show_sizes
1109 show_bloat: Show detail for each function
1110 show_config: Show config changes
1111 show_environment: Show environment changes
1113 def _BoardList(line, line_boards):
1114 """Helper function to get a line of boards containing a line
1117 line: Error line to search for
1118 line_boards: boards to search, each a Board
1120 List of boards with that error line, or [] if the user has not
1121 requested such a list
1125 if self._list_error_boards:
1126 for board in line_boards[line]:
1127 if not board in board_set:
1128 boards.append(board)
1129 board_set.add(board)
1132 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1134 """Calculate the required output based on changes in errors
1137 base_lines: List of errors/warnings for previous commit
1138 base_line_boards: Dict keyed by error line, containing a list
1139 of the Board objects with that error in the previous commit
1140 lines: List of errors/warning for this commit, each a str
1141 line_boards: Dict keyed by error line, containing a list
1142 of the Board objects with that error in this commit
1143 char: Character representing error ('') or warning ('w'). The
1144 broken ('+') or fixed ('-') characters are added in this
1149 List of ErrLine objects for 'better' lines
1150 List of ErrLine objects for 'worse' lines
1155 if line not in base_lines:
1156 errline = ErrLine(char + '+', _BoardList(line, line_boards),
1158 worse_lines.append(errline)
1159 for line in base_lines:
1160 if line not in lines:
1161 errline = ErrLine(char + '-',
1162 _BoardList(line, base_line_boards), line)
1163 better_lines.append(errline)
1164 return better_lines, worse_lines
1166 def _CalcConfig(delta, name, config):
1167 """Calculate configuration changes
1170 delta: Type of the delta, e.g. '+'
1171 name: name of the file which changed (e.g. .config)
1172 config: configuration change dictionary
1176 String containing the configuration changes which can be
1180 for key in sorted(config.keys()):
1181 out += '%s=%s ' % (key, config[key])
1182 return '%s %s: %s' % (delta, name, out)
1184 def _AddConfig(lines, name, config_plus, config_minus, config_change):
1185 """Add changes in configuration to a list
1188 lines: list to add to
1189 name: config file name
1190 config_plus: configurations added, dictionary
1193 config_minus: configurations removed, dictionary
1196 config_change: configurations changed, dictionary
1201 lines.append(_CalcConfig('+', name, config_plus))
1203 lines.append(_CalcConfig('-', name, config_minus))
1205 lines.append(_CalcConfig('c', name, config_change))
1207 def _OutputConfigInfo(lines):
1212 col = self.col.GREEN
1213 elif line[0] == '-':
1215 elif line[0] == 'c':
1216 col = self.col.YELLOW
1217 Print(' ' + line, newline=True, colour=col)
1219 def _OutputErrLines(err_lines, colour):
1220 """Output the line of error/warning lines, if not empty
1222 Also increments self._error_lines if err_lines not empty
1225 err_lines: List of ErrLine objects, each an error or warning
1226 line, possibly including a list of boards with that
1228 colour: Colour to use for output
1232 for line in err_lines:
1234 names = [board.target for board in line.boards]
1235 board_str = ' '.join(names) if names else ''
1237 out = self.col.Color(colour, line.char + '(')
1238 out += self.col.Color(self.col.MAGENTA, board_str,
1240 out += self.col.Color(colour, ') %s' % line.errline)
1242 out = self.col.Color(colour, line.char + line.errline)
1243 out_list.append(out)
1244 Print('\n'.join(out_list))
1245 self._error_lines += 1
1248 ok_boards = [] # List of boards fixed since last commit
1249 warn_boards = [] # List of boards with warnings since last commit
1250 err_boards = [] # List of new broken boards since last commit
1251 new_boards = [] # List of boards that didn't exist last time
1252 unknown_boards = [] # List of boards that were not built
1254 for target in board_dict:
1255 if target not in board_selected:
1258 # If the board was built last time, add its outcome to a list
1259 if target in self._base_board_dict:
1260 base_outcome = self._base_board_dict[target].rc
1261 outcome = board_dict[target]
1262 if outcome.rc == OUTCOME_UNKNOWN:
1263 unknown_boards.append(target)
1264 elif outcome.rc < base_outcome:
1265 if outcome.rc == OUTCOME_WARNING:
1266 warn_boards.append(target)
1268 ok_boards.append(target)
1269 elif outcome.rc > base_outcome:
1270 if outcome.rc == OUTCOME_WARNING:
1271 warn_boards.append(target)
1273 err_boards.append(target)
1275 new_boards.append(target)
1277 # Get a list of errors and warnings that have appeared, and disappeared
1278 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1279 self._base_err_line_boards, err_lines, err_line_boards, '')
1280 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1281 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1283 # Display results by arch
1284 if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1285 worse_err, better_err, worse_warn, better_warn)):
1287 self.AddOutcome(board_selected, arch_list, ok_boards, '',
1289 self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1291 self.AddOutcome(board_selected, arch_list, err_boards, '+',
1293 self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1294 if self._show_unknown:
1295 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1297 for arch, target_list in arch_list.items():
1298 Print('%10s: %s' % (arch, target_list))
1299 self._error_lines += 1
1300 _OutputErrLines(better_err, colour=self.col.GREEN)
1301 _OutputErrLines(worse_err, colour=self.col.RED)
1302 _OutputErrLines(better_warn, colour=self.col.CYAN)
1303 _OutputErrLines(worse_warn, colour=self.col.YELLOW)
1306 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1309 if show_environment and self._base_environment:
1312 for target in board_dict:
1313 if target not in board_selected:
1316 tbase = self._base_environment[target]
1317 tenvironment = environment[target]
1318 environment_plus = {}
1319 environment_minus = {}
1320 environment_change = {}
1321 base = tbase.environment
1322 for key, value in tenvironment.environment.items():
1324 environment_plus[key] = value
1325 for key, value in base.items():
1326 if key not in tenvironment.environment:
1327 environment_minus[key] = value
1328 for key, value in base.items():
1329 new_value = tenvironment.environment.get(key)
1330 if new_value and value != new_value:
1331 desc = '%s -> %s' % (value, new_value)
1332 environment_change[key] = desc
1334 _AddConfig(lines, target, environment_plus, environment_minus,
1337 _OutputConfigInfo(lines)
1339 if show_config and self._base_config:
1341 arch_config_plus = {}
1342 arch_config_minus = {}
1343 arch_config_change = {}
1346 for target in board_dict:
1347 if target not in board_selected:
1349 arch = board_selected[target].arch
1350 if arch not in arch_list:
1351 arch_list.append(arch)
1353 for arch in arch_list:
1354 arch_config_plus[arch] = {}
1355 arch_config_minus[arch] = {}
1356 arch_config_change[arch] = {}
1357 for name in self.config_filenames:
1358 arch_config_plus[arch][name] = {}
1359 arch_config_minus[arch][name] = {}
1360 arch_config_change[arch][name] = {}
1362 for target in board_dict:
1363 if target not in board_selected:
1366 arch = board_selected[target].arch
1368 all_config_plus = {}
1369 all_config_minus = {}
1370 all_config_change = {}
1371 tbase = self._base_config[target]
1372 tconfig = config[target]
1374 for name in self.config_filenames:
1375 if not tconfig.config[name]:
1380 base = tbase.config[name]
1381 for key, value in tconfig.config[name].items():
1383 config_plus[key] = value
1384 all_config_plus[key] = value
1385 for key, value in base.items():
1386 if key not in tconfig.config[name]:
1387 config_minus[key] = value
1388 all_config_minus[key] = value
1389 for key, value in base.items():
1390 new_value = tconfig.config.get(key)
1391 if new_value and value != new_value:
1392 desc = '%s -> %s' % (value, new_value)
1393 config_change[key] = desc
1394 all_config_change[key] = desc
1396 arch_config_plus[arch][name].update(config_plus)
1397 arch_config_minus[arch][name].update(config_minus)
1398 arch_config_change[arch][name].update(config_change)
1400 _AddConfig(lines, name, config_plus, config_minus,
1402 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1404 summary[target] = '\n'.join(lines)
1406 lines_by_target = {}
1407 for target, lines in summary.items():
1408 if lines in lines_by_target:
1409 lines_by_target[lines].append(target)
1411 lines_by_target[lines] = [target]
1413 for arch in arch_list:
1418 for name in self.config_filenames:
1419 all_plus.update(arch_config_plus[arch][name])
1420 all_minus.update(arch_config_minus[arch][name])
1421 all_change.update(arch_config_change[arch][name])
1422 _AddConfig(lines, name, arch_config_plus[arch][name],
1423 arch_config_minus[arch][name],
1424 arch_config_change[arch][name])
1425 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1426 #arch_summary[target] = '\n'.join(lines)
1429 _OutputConfigInfo(lines)
1431 for lines, targets in lines_by_target.items():
1434 Print('%s :' % ' '.join(sorted(targets)))
1435 _OutputConfigInfo(lines.split('\n'))
1438 # Save our updated information for the next call to this function
1439 self._base_board_dict = board_dict
1440 self._base_err_lines = err_lines
1441 self._base_warn_lines = warn_lines
1442 self._base_err_line_boards = err_line_boards
1443 self._base_warn_line_boards = warn_line_boards
1444 self._base_config = config
1445 self._base_environment = environment
1447 # Get a list of boards that did not get built, if needed
1449 for board in board_selected:
1450 if not board in board_dict:
1451 not_built.append(board)
1453 Print("Boards not built (%d): %s" % (len(not_built),
1454 ', '.join(not_built)))
1456 def ProduceResultSummary(self, commit_upto, commits, board_selected):
1457 (board_dict, err_lines, err_line_boards, warn_lines,
1458 warn_line_boards, config, environment) = self.GetResultSummary(
1459 board_selected, commit_upto,
1460 read_func_sizes=self._show_bloat,
1461 read_config=self._show_config,
1462 read_environment=self._show_environment)
1464 msg = '%02d: %s' % (commit_upto + 1,
1465 commits[commit_upto].subject)
1466 Print(msg, colour=self.col.BLUE)
1467 self.PrintResultSummary(board_selected, board_dict,
1468 err_lines if self._show_errors else [], err_line_boards,
1469 warn_lines if self._show_errors else [], warn_line_boards,
1470 config, environment, self._show_sizes, self._show_detail,
1471 self._show_bloat, self._show_config, self._show_environment)
1473 def ShowSummary(self, commits, board_selected):
1474 """Show a build summary for U-Boot for a given board list.
1476 Reset the result summary, then repeatedly call GetResultSummary on
1477 each commit's results, then display the differences we see.
1480 commit: Commit objects to summarise
1481 board_selected: Dict containing boards to summarise
1483 self.commit_count = len(commits) if commits else 1
1484 self.commits = commits
1485 self.ResetResultSummary(board_selected)
1486 self._error_lines = 0
1488 for commit_upto in range(0, self.commit_count, self._step):
1489 self.ProduceResultSummary(commit_upto, commits, board_selected)
1490 if not self._error_lines:
1491 Print('(no errors to report)', colour=self.col.GREEN)
1494 def SetupBuild(self, board_selected, commits):
1495 """Set up ready to start a build.
1498 board_selected: Selected boards to build
1499 commits: Selected commits to build
1501 # First work out how many commits we will build
1502 count = (self.commit_count + self._step - 1) // self._step
1503 self.count = len(board_selected) * count
1504 self.upto = self.warned = self.fail = 0
1505 self._timestamps = collections.deque()
1507 def GetThreadDir(self, thread_num):
1508 """Get the directory path to the working dir for a thread.
1511 thread_num: Number of thread to check.
1513 if self.work_in_output:
1514 return self._working_dir
1515 return os.path.join(self._working_dir, '%02d' % thread_num)
1517 def _PrepareThread(self, thread_num, setup_git):
1518 """Prepare the working directory for a thread.
1520 This clones or fetches the repo into the thread's work directory.
1523 thread_num: Thread number (0, 1, ...)
1524 setup_git: True to set up a git repo clone
1526 thread_dir = self.GetThreadDir(thread_num)
1527 builderthread.Mkdir(thread_dir)
1528 git_dir = os.path.join(thread_dir, '.git')
1530 # Clone the repo if it doesn't already exist
1531 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1532 # we have a private index but uses the origin repo's contents?
1533 if setup_git and self.git_dir:
1534 src_dir = os.path.abspath(self.git_dir)
1535 if os.path.exists(git_dir):
1536 gitutil.Fetch(git_dir, thread_dir)
1538 Print('\rCloning repo for thread %d' % thread_num,
1540 gitutil.Clone(src_dir, thread_dir)
1541 terminal.PrintClear()
1543 def _PrepareWorkingSpace(self, max_threads, setup_git):
1544 """Prepare the working directory for use.
1546 Set up the git repo for each thread.
1549 max_threads: Maximum number of threads we expect to need.
1550 setup_git: True to set up a git repo clone
1552 builderthread.Mkdir(self._working_dir)
1553 for thread in range(max_threads):
1554 self._PrepareThread(thread, setup_git)
1556 def _GetOutputSpaceRemovals(self):
1557 """Get the output directories ready to receive files.
1559 Figure out what needs to be deleted in the output directory before it
1560 can be used. We only delete old buildman directories which have the
1561 expected name pattern. See _GetOutputDir().
1564 List of full paths of directories to remove
1566 if not self.commits:
1569 for commit_upto in range(self.commit_count):
1570 dir_list.append(self._GetOutputDir(commit_upto))
1573 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1574 if dirname not in dir_list:
1575 leaf = dirname[len(self.base_dir) + 1:]
1576 m = re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1578 to_remove.append(dirname)
1581 def _PrepareOutputSpace(self):
1582 """Get the output directories ready to receive files.
1584 We delete any output directories which look like ones we need to
1585 create. Having left over directories is confusing when the user wants
1586 to check the output manually.
1588 to_remove = self._GetOutputSpaceRemovals()
1590 Print('Removing %d old build directories...' % len(to_remove),
1592 for dirname in to_remove:
1593 shutil.rmtree(dirname)
1596 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1597 """Build all commits for a list of boards
1600 commits: List of commits to be build, each a Commit object
1601 boards_selected: Dict of selected boards, key is target name,
1602 value is Board object
1603 keep_outputs: True to save build output files
1604 verbose: Display build results as they are completed
1607 - number of boards that failed to build
1608 - number of boards that issued warnings
1610 self.commit_count = len(commits) if commits else 1
1611 self.commits = commits
1612 self._verbose = verbose
1614 self.ResetResultSummary(board_selected)
1615 builderthread.Mkdir(self.base_dir, parents = True)
1616 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1617 commits is not None)
1618 self._PrepareOutputSpace()
1619 Print('\rStarting build...', newline=False)
1620 self.SetupBuild(board_selected, commits)
1621 self.ProcessResult(None)
1623 # Create jobs to build all commits for each board
1624 for brd in board_selected.values():
1625 job = builderthread.BuilderJob()
1627 job.commits = commits
1628 job.keep_outputs = keep_outputs
1629 job.work_in_output = self.work_in_output
1630 job.step = self._step
1633 term = threading.Thread(target=self.queue.join)
1634 term.setDaemon(True)
1636 while term.isAlive():
1639 # Wait until we have processed all output
1640 self.out_queue.join()
1642 return (self.fail, self.warned)