a5a2ffdfdf2c76d86fb78d765d9977245da652ac
[oweals/u-boot.git] / tools / buildman / builder.py
1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2013 The Chromium OS Authors.
3 #
4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5 #
6
7 import collections
8 from datetime import datetime, timedelta
9 import glob
10 import os
11 import re
12 import Queue
13 import shutil
14 import signal
15 import string
16 import sys
17 import threading
18 import time
19
20 import builderthread
21 import command
22 import gitutil
23 import terminal
24 from terminal import Print
25 import toolchain
26
27
28 """
29 Theory of Operation
30
31 Please see README for user documentation, and you should be familiar with
32 that before trying to make sense of this.
33
34 Buildman works by keeping the machine as busy as possible, building different
35 commits for different boards on multiple CPUs at once.
36
37 The source repo (self.git_dir) contains all the commits to be built. Each
38 thread works on a single board at a time. It checks out the first commit,
39 configures it for that board, then builds it. Then it checks out the next
40 commit and builds it (typically without re-configuring). When it runs out
41 of commits, it gets another job from the builder and starts again with that
42 board.
43
44 Clearly the builder threads could work either way - they could check out a
45 commit and then built it for all boards. Using separate directories for each
46 commit/board pair they could leave their build product around afterwards
47 also.
48
49 The intent behind building a single board for multiple commits, is to make
50 use of incremental builds. Since each commit is built incrementally from
51 the previous one, builds are faster. Reconfiguring for a different board
52 removes all intermediate object files.
53
54 Many threads can be working at once, but each has its own working directory.
55 When a thread finishes a build, it puts the output files into a result
56 directory.
57
58 The base directory used by buildman is normally '../<branch>', i.e.
59 a directory higher than the source repository and named after the branch
60 being built.
61
62 Within the base directory, we have one subdirectory for each commit. Within
63 that is one subdirectory for each board. Within that is the build output for
64 that commit/board combination.
65
66 Buildman also create working directories for each thread, in a .bm-work/
67 subdirectory in the base dir.
68
69 As an example, say we are building branch 'us-net' for boards 'sandbox' and
70 'seaboard', and say that us-net has two commits. We will have directories
71 like this:
72
73 us-net/             base directory
74     01_of_02_g4ed4ebc_net--Add-tftp-speed-/
75         sandbox/
76             u-boot.bin
77         seaboard/
78             u-boot.bin
79     02_of_02_g4ed4ebc_net--Check-tftp-comp/
80         sandbox/
81             u-boot.bin
82         seaboard/
83             u-boot.bin
84     .bm-work/
85         00/         working directory for thread 0 (contains source checkout)
86             build/  build output
87         01/         working directory for thread 1
88             build/  build output
89         ...
90 u-boot/             source directory
91     .git/           repository
92 """
93
94 # Possible build outcomes
95 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
96
97 # Translate a commit subject into a valid filename (and handle unicode)
98 trans_valid_chars = string.maketrans('/: ', '---')
99 trans_valid_chars = trans_valid_chars.decode('latin-1')
100
101 BASE_CONFIG_FILENAMES = [
102     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
103 ]
104
105 EXTRA_CONFIG_FILENAMES = [
106     '.config', '.config-spl', '.config-tpl',
107     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
108     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
109 ]
110
111 class Config:
112     """Holds information about configuration settings for a board."""
113     def __init__(self, config_filename, target):
114         self.target = target
115         self.config = {}
116         for fname in config_filename:
117             self.config[fname] = {}
118
119     def Add(self, fname, key, value):
120         self.config[fname][key] = value
121
122     def __hash__(self):
123         val = 0
124         for fname in self.config:
125             for key, value in self.config[fname].iteritems():
126                 print key, value
127                 val = val ^ hash(key) & hash(value)
128         return val
129
130 class Environment:
131     """Holds information about environment variables for a board."""
132     def __init__(self, target):
133         self.target = target
134         self.environment = {}
135
136     def Add(self, key, value):
137         self.environment[key] = value
138
139 class Builder:
140     """Class for building U-Boot for a particular commit.
141
142     Public members: (many should ->private)
143         already_done: Number of builds already completed
144         base_dir: Base directory to use for builder
145         checkout: True to check out source, False to skip that step.
146             This is used for testing.
147         col: terminal.Color() object
148         count: Number of commits to build
149         do_make: Method to call to invoke Make
150         fail: Number of builds that failed due to error
151         force_build: Force building even if a build already exists
152         force_config_on_failure: If a commit fails for a board, disable
153             incremental building for the next commit we build for that
154             board, so that we will see all warnings/errors again.
155         force_build_failures: If a previously-built build (i.e. built on
156             a previous run of buildman) is marked as failed, rebuild it.
157         git_dir: Git directory containing source repository
158         last_line_len: Length of the last line we printed (used for erasing
159             it with new progress information)
160         num_jobs: Number of jobs to run at once (passed to make as -j)
161         num_threads: Number of builder threads to run
162         out_queue: Queue of results to process
163         re_make_err: Compiled regular expression for ignore_lines
164         queue: Queue of jobs to run
165         threads: List of active threads
166         toolchains: Toolchains object to use for building
167         upto: Current commit number we are building (0.count-1)
168         warned: Number of builds that produced at least one warning
169         force_reconfig: Reconfigure U-Boot on each comiit. This disables
170             incremental building, where buildman reconfigures on the first
171             commit for a baord, and then just does an incremental build for
172             the following commits. In fact buildman will reconfigure and
173             retry for any failing commits, so generally the only effect of
174             this option is to slow things down.
175         in_tree: Build U-Boot in-tree instead of specifying an output
176             directory separate from the source code. This option is really
177             only useful for testing in-tree builds.
178
179     Private members:
180         _base_board_dict: Last-summarised Dict of boards
181         _base_err_lines: Last-summarised list of errors
182         _base_warn_lines: Last-summarised list of warnings
183         _build_period_us: Time taken for a single build (float object).
184         _complete_delay: Expected delay until completion (timedelta)
185         _next_delay_update: Next time we plan to display a progress update
186                 (datatime)
187         _show_unknown: Show unknown boards (those not built) in summary
188         _timestamps: List of timestamps for the completion of the last
189             last _timestamp_count builds. Each is a datetime object.
190         _timestamp_count: Number of timestamps to keep in our list.
191         _working_dir: Base working directory containing all threads
192     """
193     class Outcome:
194         """Records a build outcome for a single make invocation
195
196         Public Members:
197             rc: Outcome value (OUTCOME_...)
198             err_lines: List of error lines or [] if none
199             sizes: Dictionary of image size information, keyed by filename
200                 - Each value is itself a dictionary containing
201                     values for 'text', 'data' and 'bss', being the integer
202                     size in bytes of each section.
203             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
204                     value is itself a dictionary:
205                         key: function name
206                         value: Size of function in bytes
207             config: Dictionary keyed by filename - e.g. '.config'. Each
208                     value is itself a dictionary:
209                         key: config name
210                         value: config value
211             environment: Dictionary keyed by environment variable, Each
212                      value is the value of environment variable.
213         """
214         def __init__(self, rc, err_lines, sizes, func_sizes, config,
215                      environment):
216             self.rc = rc
217             self.err_lines = err_lines
218             self.sizes = sizes
219             self.func_sizes = func_sizes
220             self.config = config
221             self.environment = environment
222
223     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
224                  gnu_make='make', checkout=True, show_unknown=True, step=1,
225                  no_subdirs=False, full_path=False, verbose_build=False,
226                  incremental=False, per_board_out_dir=False,
227                  config_only=False, squash_config_y=False,
228                  warnings_as_errors=False):
229         """Create a new Builder object
230
231         Args:
232             toolchains: Toolchains object to use for building
233             base_dir: Base directory to use for builder
234             git_dir: Git directory containing source repository
235             num_threads: Number of builder threads to run
236             num_jobs: Number of jobs to run at once (passed to make as -j)
237             gnu_make: the command name of GNU Make.
238             checkout: True to check out source, False to skip that step.
239                 This is used for testing.
240             show_unknown: Show unknown boards (those not built) in summary
241             step: 1 to process every commit, n to process every nth commit
242             no_subdirs: Don't create subdirectories when building current
243                 source for a single board
244             full_path: Return the full path in CROSS_COMPILE and don't set
245                 PATH
246             verbose_build: Run build with V=1 and don't use 'make -s'
247             incremental: Always perform incremental builds; don't run make
248                 mrproper when configuring
249             per_board_out_dir: Build in a separate persistent directory per
250                 board rather than a thread-specific directory
251             config_only: Only configure each build, don't build it
252             squash_config_y: Convert CONFIG options with the value 'y' to '1'
253             warnings_as_errors: Treat all compiler warnings as errors
254         """
255         self.toolchains = toolchains
256         self.base_dir = base_dir
257         self._working_dir = os.path.join(base_dir, '.bm-work')
258         self.threads = []
259         self.do_make = self.Make
260         self.gnu_make = gnu_make
261         self.checkout = checkout
262         self.num_threads = num_threads
263         self.num_jobs = num_jobs
264         self.already_done = 0
265         self.force_build = False
266         self.git_dir = git_dir
267         self._show_unknown = show_unknown
268         self._timestamp_count = 10
269         self._build_period_us = None
270         self._complete_delay = None
271         self._next_delay_update = datetime.now()
272         self.force_config_on_failure = True
273         self.force_build_failures = False
274         self.force_reconfig = False
275         self._step = step
276         self.in_tree = False
277         self._error_lines = 0
278         self.no_subdirs = no_subdirs
279         self.full_path = full_path
280         self.verbose_build = verbose_build
281         self.config_only = config_only
282         self.squash_config_y = squash_config_y
283         self.config_filenames = BASE_CONFIG_FILENAMES
284         if not self.squash_config_y:
285             self.config_filenames += EXTRA_CONFIG_FILENAMES
286
287         self.warnings_as_errors = warnings_as_errors
288         self.col = terminal.Color()
289
290         self._re_function = re.compile('(.*): In function.*')
291         self._re_files = re.compile('In file included from.*')
292         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
293         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
294
295         self.queue = Queue.Queue()
296         self.out_queue = Queue.Queue()
297         for i in range(self.num_threads):
298             t = builderthread.BuilderThread(self, i, incremental,
299                     per_board_out_dir)
300             t.setDaemon(True)
301             t.start()
302             self.threads.append(t)
303
304         self.last_line_len = 0
305         t = builderthread.ResultThread(self)
306         t.setDaemon(True)
307         t.start()
308         self.threads.append(t)
309
310         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
311         self.re_make_err = re.compile('|'.join(ignore_lines))
312
313         # Handle existing graceful with SIGINT / Ctrl-C
314         signal.signal(signal.SIGINT, self.signal_handler)
315
316     def __del__(self):
317         """Get rid of all threads created by the builder"""
318         for t in self.threads:
319             del t
320
321     def signal_handler(self, signal, frame):
322         sys.exit(1)
323
324     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
325                           show_detail=False, show_bloat=False,
326                           list_error_boards=False, show_config=False,
327                           show_environment=False):
328         """Setup display options for the builder.
329
330         show_errors: True to show summarised error/warning info
331         show_sizes: Show size deltas
332         show_detail: Show detail for each board
333         show_bloat: Show detail for each function
334         list_error_boards: Show the boards which caused each error/warning
335         show_config: Show config deltas
336         show_environment: Show environment deltas
337         """
338         self._show_errors = show_errors
339         self._show_sizes = show_sizes
340         self._show_detail = show_detail
341         self._show_bloat = show_bloat
342         self._list_error_boards = list_error_boards
343         self._show_config = show_config
344         self._show_environment = show_environment
345
346     def _AddTimestamp(self):
347         """Add a new timestamp to the list and record the build period.
348
349         The build period is the length of time taken to perform a single
350         build (one board, one commit).
351         """
352         now = datetime.now()
353         self._timestamps.append(now)
354         count = len(self._timestamps)
355         delta = self._timestamps[-1] - self._timestamps[0]
356         seconds = delta.total_seconds()
357
358         # If we have enough data, estimate build period (time taken for a
359         # single build) and therefore completion time.
360         if count > 1 and self._next_delay_update < now:
361             self._next_delay_update = now + timedelta(seconds=2)
362             if seconds > 0:
363                 self._build_period = float(seconds) / count
364                 todo = self.count - self.upto
365                 self._complete_delay = timedelta(microseconds=
366                         self._build_period * todo * 1000000)
367                 # Round it
368                 self._complete_delay -= timedelta(
369                         microseconds=self._complete_delay.microseconds)
370
371         if seconds > 60:
372             self._timestamps.popleft()
373             count -= 1
374
375     def ClearLine(self, length):
376         """Clear any characters on the current line
377
378         Make way for a new line of length 'length', by outputting enough
379         spaces to clear out the old line. Then remember the new length for
380         next time.
381
382         Args:
383             length: Length of new line, in characters
384         """
385         if length < self.last_line_len:
386             Print(' ' * (self.last_line_len - length), newline=False)
387             Print('\r', newline=False)
388         self.last_line_len = length
389         sys.stdout.flush()
390
391     def SelectCommit(self, commit, checkout=True):
392         """Checkout the selected commit for this build
393         """
394         self.commit = commit
395         if checkout and self.checkout:
396             gitutil.Checkout(commit.hash)
397
398     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
399         """Run make
400
401         Args:
402             commit: Commit object that is being built
403             brd: Board object that is being built
404             stage: Stage that we are at (mrproper, config, build)
405             cwd: Directory where make should be run
406             args: Arguments to pass to make
407             kwargs: Arguments to pass to command.RunPipe()
408         """
409         cmd = [self.gnu_make] + list(args)
410         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
411                 cwd=cwd, raise_on_error=False, **kwargs)
412         if self.verbose_build:
413             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
414             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
415         return result
416
417     def ProcessResult(self, result):
418         """Process the result of a build, showing progress information
419
420         Args:
421             result: A CommandResult object, which indicates the result for
422                     a single build
423         """
424         col = terminal.Color()
425         if result:
426             target = result.brd.target
427
428             self.upto += 1
429             if result.return_code != 0:
430                 self.fail += 1
431             elif result.stderr:
432                 self.warned += 1
433             if result.already_done:
434                 self.already_done += 1
435             if self._verbose:
436                 Print('\r', newline=False)
437                 self.ClearLine(0)
438                 boards_selected = {target : result.brd}
439                 self.ResetResultSummary(boards_selected)
440                 self.ProduceResultSummary(result.commit_upto, self.commits,
441                                           boards_selected)
442         else:
443             target = '(starting)'
444
445         # Display separate counts for ok, warned and fail
446         ok = self.upto - self.warned - self.fail
447         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
448         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
449         line += self.col.Color(self.col.RED, '%5d' % self.fail)
450
451         name = ' /%-5d  ' % self.count
452
453         # Add our current completion time estimate
454         self._AddTimestamp()
455         if self._complete_delay:
456             name += '%s  : ' % self._complete_delay
457         # When building all boards for a commit, we can print a commit
458         # progress message.
459         if result and result.commit_upto is None:
460             name += 'commit %2d/%-3d' % (self.commit_upto + 1,
461                     self.commit_count)
462
463         name += target
464         Print(line + name, newline=False)
465         length = 16 + len(name)
466         self.ClearLine(length)
467
468     def _GetOutputDir(self, commit_upto):
469         """Get the name of the output directory for a commit number
470
471         The output directory is typically .../<branch>/<commit>.
472
473         Args:
474             commit_upto: Commit number to use (0..self.count-1)
475         """
476         commit_dir = None
477         if self.commits:
478             commit = self.commits[commit_upto]
479             subject = commit.subject.translate(trans_valid_chars)
480             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
481                     self.commit_count, commit.hash, subject[:20]))
482         elif not self.no_subdirs:
483             commit_dir = 'current'
484         if not commit_dir:
485             return self.base_dir
486         return os.path.join(self.base_dir, commit_dir)
487
488     def GetBuildDir(self, commit_upto, target):
489         """Get the name of the build directory for a commit number
490
491         The build directory is typically .../<branch>/<commit>/<target>.
492
493         Args:
494             commit_upto: Commit number to use (0..self.count-1)
495             target: Target name
496         """
497         output_dir = self._GetOutputDir(commit_upto)
498         return os.path.join(output_dir, target)
499
500     def GetDoneFile(self, commit_upto, target):
501         """Get the name of the done file for a commit number
502
503         Args:
504             commit_upto: Commit number to use (0..self.count-1)
505             target: Target name
506         """
507         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
508
509     def GetSizesFile(self, commit_upto, target):
510         """Get the name of the sizes file for a commit number
511
512         Args:
513             commit_upto: Commit number to use (0..self.count-1)
514             target: Target name
515         """
516         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
517
518     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
519         """Get the name of the funcsizes file for a commit number and ELF file
520
521         Args:
522             commit_upto: Commit number to use (0..self.count-1)
523             target: Target name
524             elf_fname: Filename of elf image
525         """
526         return os.path.join(self.GetBuildDir(commit_upto, target),
527                             '%s.sizes' % elf_fname.replace('/', '-'))
528
529     def GetObjdumpFile(self, commit_upto, target, elf_fname):
530         """Get the name of the objdump file for a commit number and ELF file
531
532         Args:
533             commit_upto: Commit number to use (0..self.count-1)
534             target: Target name
535             elf_fname: Filename of elf image
536         """
537         return os.path.join(self.GetBuildDir(commit_upto, target),
538                             '%s.objdump' % elf_fname.replace('/', '-'))
539
540     def GetErrFile(self, commit_upto, target):
541         """Get the name of the err file for a commit number
542
543         Args:
544             commit_upto: Commit number to use (0..self.count-1)
545             target: Target name
546         """
547         output_dir = self.GetBuildDir(commit_upto, target)
548         return os.path.join(output_dir, 'err')
549
550     def FilterErrors(self, lines):
551         """Filter out errors in which we have no interest
552
553         We should probably use map().
554
555         Args:
556             lines: List of error lines, each a string
557         Returns:
558             New list with only interesting lines included
559         """
560         out_lines = []
561         for line in lines:
562             if not self.re_make_err.search(line):
563                 out_lines.append(line)
564         return out_lines
565
566     def ReadFuncSizes(self, fname, fd):
567         """Read function sizes from the output of 'nm'
568
569         Args:
570             fd: File containing data to read
571             fname: Filename we are reading from (just for errors)
572
573         Returns:
574             Dictionary containing size of each function in bytes, indexed by
575             function name.
576         """
577         sym = {}
578         for line in fd.readlines():
579             try:
580                 size, type, name = line[:-1].split()
581             except:
582                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
583                 continue
584             if type in 'tTdDbB':
585                 # function names begin with '.' on 64-bit powerpc
586                 if '.' in name[1:]:
587                     name = 'static.' + name.split('.')[0]
588                 sym[name] = sym.get(name, 0) + int(size, 16)
589         return sym
590
591     def _ProcessConfig(self, fname):
592         """Read in a .config, autoconf.mk or autoconf.h file
593
594         This function handles all config file types. It ignores comments and
595         any #defines which don't start with CONFIG_.
596
597         Args:
598             fname: Filename to read
599
600         Returns:
601             Dictionary:
602                 key: Config name (e.g. CONFIG_DM)
603                 value: Config value (e.g. 1)
604         """
605         config = {}
606         if os.path.exists(fname):
607             with open(fname) as fd:
608                 for line in fd:
609                     line = line.strip()
610                     if line.startswith('#define'):
611                         values = line[8:].split(' ', 1)
612                         if len(values) > 1:
613                             key, value = values
614                         else:
615                             key = values[0]
616                             value = '1' if self.squash_config_y else ''
617                         if not key.startswith('CONFIG_'):
618                             continue
619                     elif not line or line[0] in ['#', '*', '/']:
620                         continue
621                     else:
622                         key, value = line.split('=', 1)
623                     if self.squash_config_y and value == 'y':
624                         value = '1'
625                     config[key] = value
626         return config
627
628     def _ProcessEnvironment(self, fname):
629         """Read in a uboot.env file
630
631         This function reads in environment variables from a file.
632
633         Args:
634             fname: Filename to read
635
636         Returns:
637             Dictionary:
638                 key: environment variable (e.g. bootlimit)
639                 value: value of environment variable (e.g. 1)
640         """
641         environment = {}
642         if os.path.exists(fname):
643             with open(fname) as fd:
644                 for line in fd.read().split('\0'):
645                     try:
646                         key, value = line.split('=', 1)
647                         environment[key] = value
648                     except ValueError:
649                         # ignore lines we can't parse
650                         pass
651         return environment
652
653     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
654                         read_config, read_environment):
655         """Work out the outcome of a build.
656
657         Args:
658             commit_upto: Commit number to check (0..n-1)
659             target: Target board to check
660             read_func_sizes: True to read function size information
661             read_config: True to read .config and autoconf.h files
662             read_environment: True to read uboot.env files
663
664         Returns:
665             Outcome object
666         """
667         done_file = self.GetDoneFile(commit_upto, target)
668         sizes_file = self.GetSizesFile(commit_upto, target)
669         sizes = {}
670         func_sizes = {}
671         config = {}
672         environment = {}
673         if os.path.exists(done_file):
674             with open(done_file, 'r') as fd:
675                 return_code = int(fd.readline())
676                 err_lines = []
677                 err_file = self.GetErrFile(commit_upto, target)
678                 if os.path.exists(err_file):
679                     with open(err_file, 'r') as fd:
680                         err_lines = self.FilterErrors(fd.readlines())
681
682                 # Decide whether the build was ok, failed or created warnings
683                 if return_code:
684                     rc = OUTCOME_ERROR
685                 elif len(err_lines):
686                     rc = OUTCOME_WARNING
687                 else:
688                     rc = OUTCOME_OK
689
690                 # Convert size information to our simple format
691                 if os.path.exists(sizes_file):
692                     with open(sizes_file, 'r') as fd:
693                         for line in fd.readlines():
694                             values = line.split()
695                             rodata = 0
696                             if len(values) > 6:
697                                 rodata = int(values[6], 16)
698                             size_dict = {
699                                 'all' : int(values[0]) + int(values[1]) +
700                                         int(values[2]),
701                                 'text' : int(values[0]) - rodata,
702                                 'data' : int(values[1]),
703                                 'bss' : int(values[2]),
704                                 'rodata' : rodata,
705                             }
706                             sizes[values[5]] = size_dict
707
708             if read_func_sizes:
709                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
710                 for fname in glob.glob(pattern):
711                     with open(fname, 'r') as fd:
712                         dict_name = os.path.basename(fname).replace('.sizes',
713                                                                     '')
714                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
715
716             if read_config:
717                 output_dir = self.GetBuildDir(commit_upto, target)
718                 for name in self.config_filenames:
719                     fname = os.path.join(output_dir, name)
720                     config[name] = self._ProcessConfig(fname)
721
722             if read_environment:
723                 output_dir = self.GetBuildDir(commit_upto, target)
724                 fname = os.path.join(output_dir, 'uboot.env')
725                 environment = self._ProcessEnvironment(fname)
726
727             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
728                                    environment)
729
730         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
731
732     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
733                          read_config, read_environment):
734         """Calculate a summary of the results of building a commit.
735
736         Args:
737             board_selected: Dict containing boards to summarise
738             commit_upto: Commit number to summarize (0..self.count-1)
739             read_func_sizes: True to read function size information
740             read_config: True to read .config and autoconf.h files
741             read_environment: True to read uboot.env files
742
743         Returns:
744             Tuple:
745                 Dict containing boards which passed building this commit.
746                     keyed by board.target
747                 List containing a summary of error lines
748                 Dict keyed by error line, containing a list of the Board
749                     objects with that error
750                 List containing a summary of warning lines
751                 Dict keyed by error line, containing a list of the Board
752                     objects with that warning
753                 Dictionary keyed by board.target. Each value is a dictionary:
754                     key: filename - e.g. '.config'
755                     value is itself a dictionary:
756                         key: config name
757                         value: config value
758                 Dictionary keyed by board.target. Each value is a dictionary:
759                     key: environment variable
760                     value: value of environment variable
761         """
762         def AddLine(lines_summary, lines_boards, line, board):
763             line = line.rstrip()
764             if line in lines_boards:
765                 lines_boards[line].append(board)
766             else:
767                 lines_boards[line] = [board]
768                 lines_summary.append(line)
769
770         board_dict = {}
771         err_lines_summary = []
772         err_lines_boards = {}
773         warn_lines_summary = []
774         warn_lines_boards = {}
775         config = {}
776         environment = {}
777
778         for board in boards_selected.itervalues():
779             outcome = self.GetBuildOutcome(commit_upto, board.target,
780                                            read_func_sizes, read_config,
781                                            read_environment)
782             board_dict[board.target] = outcome
783             last_func = None
784             last_was_warning = False
785             for line in outcome.err_lines:
786                 if line:
787                     if (self._re_function.match(line) or
788                             self._re_files.match(line)):
789                         last_func = line
790                     else:
791                         is_warning = self._re_warning.match(line)
792                         is_note = self._re_note.match(line)
793                         if is_warning or (last_was_warning and is_note):
794                             if last_func:
795                                 AddLine(warn_lines_summary, warn_lines_boards,
796                                         last_func, board)
797                             AddLine(warn_lines_summary, warn_lines_boards,
798                                     line, board)
799                         else:
800                             if last_func:
801                                 AddLine(err_lines_summary, err_lines_boards,
802                                         last_func, board)
803                             AddLine(err_lines_summary, err_lines_boards,
804                                     line, board)
805                         last_was_warning = is_warning
806                         last_func = None
807             tconfig = Config(self.config_filenames, board.target)
808             for fname in self.config_filenames:
809                 if outcome.config:
810                     for key, value in outcome.config[fname].iteritems():
811                         tconfig.Add(fname, key, value)
812             config[board.target] = tconfig
813
814             tenvironment = Environment(board.target)
815             if outcome.environment:
816                 for key, value in outcome.environment.iteritems():
817                     tenvironment.Add(key, value)
818             environment[board.target] = tenvironment
819
820         return (board_dict, err_lines_summary, err_lines_boards,
821                 warn_lines_summary, warn_lines_boards, config, environment)
822
823     def AddOutcome(self, board_dict, arch_list, changes, char, color):
824         """Add an output to our list of outcomes for each architecture
825
826         This simple function adds failing boards (changes) to the
827         relevant architecture string, so we can print the results out
828         sorted by architecture.
829
830         Args:
831              board_dict: Dict containing all boards
832              arch_list: Dict keyed by arch name. Value is a string containing
833                     a list of board names which failed for that arch.
834              changes: List of boards to add to arch_list
835              color: terminal.Colour object
836         """
837         done_arch = {}
838         for target in changes:
839             if target in board_dict:
840                 arch = board_dict[target].arch
841             else:
842                 arch = 'unknown'
843             str = self.col.Color(color, ' ' + target)
844             if not arch in done_arch:
845                 str = ' %s  %s' % (self.col.Color(color, char), str)
846                 done_arch[arch] = True
847             if not arch in arch_list:
848                 arch_list[arch] = str
849             else:
850                 arch_list[arch] += str
851
852
853     def ColourNum(self, num):
854         color = self.col.RED if num > 0 else self.col.GREEN
855         if num == 0:
856             return '0'
857         return self.col.Color(color, str(num))
858
859     def ResetResultSummary(self, board_selected):
860         """Reset the results summary ready for use.
861
862         Set up the base board list to be all those selected, and set the
863         error lines to empty.
864
865         Following this, calls to PrintResultSummary() will use this
866         information to work out what has changed.
867
868         Args:
869             board_selected: Dict containing boards to summarise, keyed by
870                 board.target
871         """
872         self._base_board_dict = {}
873         for board in board_selected:
874             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
875                                                            {})
876         self._base_err_lines = []
877         self._base_warn_lines = []
878         self._base_err_line_boards = {}
879         self._base_warn_line_boards = {}
880         self._base_config = None
881         self._base_environment = None
882
883     def PrintFuncSizeDetail(self, fname, old, new):
884         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
885         delta, common = [], {}
886
887         for a in old:
888             if a in new:
889                 common[a] = 1
890
891         for name in old:
892             if name not in common:
893                 remove += 1
894                 down += old[name]
895                 delta.append([-old[name], name])
896
897         for name in new:
898             if name not in common:
899                 add += 1
900                 up += new[name]
901                 delta.append([new[name], name])
902
903         for name in common:
904                 diff = new.get(name, 0) - old.get(name, 0)
905                 if diff > 0:
906                     grow, up = grow + 1, up + diff
907                 elif diff < 0:
908                     shrink, down = shrink + 1, down - diff
909                 delta.append([diff, name])
910
911         delta.sort()
912         delta.reverse()
913
914         args = [add, -remove, grow, -shrink, up, -down, up - down]
915         if max(args) == 0 and min(args) == 0:
916             return
917         args = [self.ColourNum(x) for x in args]
918         indent = ' ' * 15
919         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
920               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
921         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
922                                          'delta'))
923         for diff, name in delta:
924             if diff:
925                 color = self.col.RED if diff > 0 else self.col.GREEN
926                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
927                         old.get(name, '-'), new.get(name,'-'), diff)
928                 Print(msg, colour=color)
929
930
931     def PrintSizeDetail(self, target_list, show_bloat):
932         """Show details size information for each board
933
934         Args:
935             target_list: List of targets, each a dict containing:
936                     'target': Target name
937                     'total_diff': Total difference in bytes across all areas
938                     <part_name>: Difference for that part
939             show_bloat: Show detail for each function
940         """
941         targets_by_diff = sorted(target_list, reverse=True,
942         key=lambda x: x['_total_diff'])
943         for result in targets_by_diff:
944             printed_target = False
945             for name in sorted(result):
946                 diff = result[name]
947                 if name.startswith('_'):
948                     continue
949                 if diff != 0:
950                     color = self.col.RED if diff > 0 else self.col.GREEN
951                 msg = ' %s %+d' % (name, diff)
952                 if not printed_target:
953                     Print('%10s  %-15s:' % ('', result['_target']),
954                           newline=False)
955                     printed_target = True
956                 Print(msg, colour=color, newline=False)
957             if printed_target:
958                 Print()
959                 if show_bloat:
960                     target = result['_target']
961                     outcome = result['_outcome']
962                     base_outcome = self._base_board_dict[target]
963                     for fname in outcome.func_sizes:
964                         self.PrintFuncSizeDetail(fname,
965                                                  base_outcome.func_sizes[fname],
966                                                  outcome.func_sizes[fname])
967
968
969     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
970                          show_bloat):
971         """Print a summary of image sizes broken down by section.
972
973         The summary takes the form of one line per architecture. The
974         line contains deltas for each of the sections (+ means the section
975         got bigger, - means smaller). The nunmbers are the average number
976         of bytes that a board in this section increased by.
977
978         For example:
979            powerpc: (622 boards)   text -0.0
980           arm: (285 boards)   text -0.0
981           nds32: (3 boards)   text -8.0
982
983         Args:
984             board_selected: Dict containing boards to summarise, keyed by
985                 board.target
986             board_dict: Dict containing boards for which we built this
987                 commit, keyed by board.target. The value is an Outcome object.
988             show_detail: Show detail for each board
989             show_bloat: Show detail for each function
990         """
991         arch_list = {}
992         arch_count = {}
993
994         # Calculate changes in size for different image parts
995         # The previous sizes are in Board.sizes, for each board
996         for target in board_dict:
997             if target not in board_selected:
998                 continue
999             base_sizes = self._base_board_dict[target].sizes
1000             outcome = board_dict[target]
1001             sizes = outcome.sizes
1002
1003             # Loop through the list of images, creating a dict of size
1004             # changes for each image/part. We end up with something like
1005             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1006             # which means that U-Boot data increased by 5 bytes and SPL
1007             # text decreased by 4.
1008             err = {'_target' : target}
1009             for image in sizes:
1010                 if image in base_sizes:
1011                     base_image = base_sizes[image]
1012                     # Loop through the text, data, bss parts
1013                     for part in sorted(sizes[image]):
1014                         diff = sizes[image][part] - base_image[part]
1015                         col = None
1016                         if diff:
1017                             if image == 'u-boot':
1018                                 name = part
1019                             else:
1020                                 name = image + ':' + part
1021                             err[name] = diff
1022             arch = board_selected[target].arch
1023             if not arch in arch_count:
1024                 arch_count[arch] = 1
1025             else:
1026                 arch_count[arch] += 1
1027             if not sizes:
1028                 pass    # Only add to our list when we have some stats
1029             elif not arch in arch_list:
1030                 arch_list[arch] = [err]
1031             else:
1032                 arch_list[arch].append(err)
1033
1034         # We now have a list of image size changes sorted by arch
1035         # Print out a summary of these
1036         for arch, target_list in arch_list.iteritems():
1037             # Get total difference for each type
1038             totals = {}
1039             for result in target_list:
1040                 total = 0
1041                 for name, diff in result.iteritems():
1042                     if name.startswith('_'):
1043                         continue
1044                     total += diff
1045                     if name in totals:
1046                         totals[name] += diff
1047                     else:
1048                         totals[name] = diff
1049                 result['_total_diff'] = total
1050                 result['_outcome'] = board_dict[result['_target']]
1051
1052             count = len(target_list)
1053             printed_arch = False
1054             for name in sorted(totals):
1055                 diff = totals[name]
1056                 if diff:
1057                     # Display the average difference in this name for this
1058                     # architecture
1059                     avg_diff = float(diff) / count
1060                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1061                     msg = ' %s %+1.1f' % (name, avg_diff)
1062                     if not printed_arch:
1063                         Print('%10s: (for %d/%d boards)' % (arch, count,
1064                               arch_count[arch]), newline=False)
1065                         printed_arch = True
1066                     Print(msg, colour=color, newline=False)
1067
1068             if printed_arch:
1069                 Print()
1070                 if show_detail:
1071                     self.PrintSizeDetail(target_list, show_bloat)
1072
1073
1074     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1075                            err_line_boards, warn_lines, warn_line_boards,
1076                            config, environment, show_sizes, show_detail,
1077                            show_bloat, show_config, show_environment):
1078         """Compare results with the base results and display delta.
1079
1080         Only boards mentioned in board_selected will be considered. This
1081         function is intended to be called repeatedly with the results of
1082         each commit. It therefore shows a 'diff' between what it saw in
1083         the last call and what it sees now.
1084
1085         Args:
1086             board_selected: Dict containing boards to summarise, keyed by
1087                 board.target
1088             board_dict: Dict containing boards for which we built this
1089                 commit, keyed by board.target. The value is an Outcome object.
1090             err_lines: A list of errors for this commit, or [] if there is
1091                 none, or we don't want to print errors
1092             err_line_boards: Dict keyed by error line, containing a list of
1093                 the Board objects with that error
1094             warn_lines: A list of warnings for this commit, or [] if there is
1095                 none, or we don't want to print errors
1096             warn_line_boards: Dict keyed by warning line, containing a list of
1097                 the Board objects with that warning
1098             config: Dictionary keyed by filename - e.g. '.config'. Each
1099                     value is itself a dictionary:
1100                         key: config name
1101                         value: config value
1102             environment: Dictionary keyed by environment variable, Each
1103                      value is the value of environment variable.
1104             show_sizes: Show image size deltas
1105             show_detail: Show detail for each board
1106             show_bloat: Show detail for each function
1107             show_config: Show config changes
1108             show_environment: Show environment changes
1109         """
1110         def _BoardList(line, line_boards):
1111             """Helper function to get a line of boards containing a line
1112
1113             Args:
1114                 line: Error line to search for
1115             Return:
1116                 String containing a list of boards with that error line, or
1117                 '' if the user has not requested such a list
1118             """
1119             if self._list_error_boards:
1120                 names = []
1121                 for board in line_boards[line]:
1122                     if not board.target in names:
1123                         names.append(board.target)
1124                 names_str = '(%s) ' % ','.join(names)
1125             else:
1126                 names_str = ''
1127             return names_str
1128
1129         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1130                             char):
1131             better_lines = []
1132             worse_lines = []
1133             for line in lines:
1134                 if line not in base_lines:
1135                     worse_lines.append(char + '+' +
1136                             _BoardList(line, line_boards) + line)
1137             for line in base_lines:
1138                 if line not in lines:
1139                     better_lines.append(char + '-' +
1140                             _BoardList(line, base_line_boards) + line)
1141             return better_lines, worse_lines
1142
1143         def _CalcConfig(delta, name, config):
1144             """Calculate configuration changes
1145
1146             Args:
1147                 delta: Type of the delta, e.g. '+'
1148                 name: name of the file which changed (e.g. .config)
1149                 config: configuration change dictionary
1150                     key: config name
1151                     value: config value
1152             Returns:
1153                 String containing the configuration changes which can be
1154                     printed
1155             """
1156             out = ''
1157             for key in sorted(config.keys()):
1158                 out += '%s=%s ' % (key, config[key])
1159             return '%s %s: %s' % (delta, name, out)
1160
1161         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1162             """Add changes in configuration to a list
1163
1164             Args:
1165                 lines: list to add to
1166                 name: config file name
1167                 config_plus: configurations added, dictionary
1168                     key: config name
1169                     value: config value
1170                 config_minus: configurations removed, dictionary
1171                     key: config name
1172                     value: config value
1173                 config_change: configurations changed, dictionary
1174                     key: config name
1175                     value: config value
1176             """
1177             if config_plus:
1178                 lines.append(_CalcConfig('+', name, config_plus))
1179             if config_minus:
1180                 lines.append(_CalcConfig('-', name, config_minus))
1181             if config_change:
1182                 lines.append(_CalcConfig('c', name, config_change))
1183
1184         def _OutputConfigInfo(lines):
1185             for line in lines:
1186                 if not line:
1187                     continue
1188                 if line[0] == '+':
1189                     col = self.col.GREEN
1190                 elif line[0] == '-':
1191                     col = self.col.RED
1192                 elif line[0] == 'c':
1193                     col = self.col.YELLOW
1194                 Print('   ' + line, newline=True, colour=col)
1195
1196
1197         better = []     # List of boards fixed since last commit
1198         worse = []      # List of new broken boards since last commit
1199         new = []        # List of boards that didn't exist last time
1200         unknown = []    # List of boards that were not built
1201
1202         for target in board_dict:
1203             if target not in board_selected:
1204                 continue
1205
1206             # If the board was built last time, add its outcome to a list
1207             if target in self._base_board_dict:
1208                 base_outcome = self._base_board_dict[target].rc
1209                 outcome = board_dict[target]
1210                 if outcome.rc == OUTCOME_UNKNOWN:
1211                     unknown.append(target)
1212                 elif outcome.rc < base_outcome:
1213                     better.append(target)
1214                 elif outcome.rc > base_outcome:
1215                     worse.append(target)
1216             else:
1217                 new.append(target)
1218
1219         # Get a list of errors that have appeared, and disappeared
1220         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1221                 self._base_err_line_boards, err_lines, err_line_boards, '')
1222         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1223                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1224
1225         # Display results by arch
1226         if (better or worse or unknown or new or worse_err or better_err
1227                 or worse_warn or better_warn):
1228             arch_list = {}
1229             self.AddOutcome(board_selected, arch_list, better, '',
1230                     self.col.GREEN)
1231             self.AddOutcome(board_selected, arch_list, worse, '+',
1232                     self.col.RED)
1233             self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
1234             if self._show_unknown:
1235                 self.AddOutcome(board_selected, arch_list, unknown, '?',
1236                         self.col.MAGENTA)
1237             for arch, target_list in arch_list.iteritems():
1238                 Print('%10s: %s' % (arch, target_list))
1239                 self._error_lines += 1
1240             if better_err:
1241                 Print('\n'.join(better_err), colour=self.col.GREEN)
1242                 self._error_lines += 1
1243             if worse_err:
1244                 Print('\n'.join(worse_err), colour=self.col.RED)
1245                 self._error_lines += 1
1246             if better_warn:
1247                 Print('\n'.join(better_warn), colour=self.col.CYAN)
1248                 self._error_lines += 1
1249             if worse_warn:
1250                 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
1251                 self._error_lines += 1
1252
1253         if show_sizes:
1254             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1255                                   show_bloat)
1256
1257         if show_environment and self._base_environment:
1258             lines = []
1259
1260             for target in board_dict:
1261                 if target not in board_selected:
1262                     continue
1263
1264                 tbase = self._base_environment[target]
1265                 tenvironment = environment[target]
1266                 environment_plus = {}
1267                 environment_minus = {}
1268                 environment_change = {}
1269                 base = tbase.environment
1270                 for key, value in tenvironment.environment.iteritems():
1271                     if key not in base:
1272                         environment_plus[key] = value
1273                 for key, value in base.iteritems():
1274                     if key not in tenvironment.environment:
1275                         environment_minus[key] = value
1276                 for key, value in base.iteritems():
1277                     new_value = tenvironment.environment.get(key)
1278                     if new_value and value != new_value:
1279                         desc = '%s -> %s' % (value, new_value)
1280                         environment_change[key] = desc
1281
1282                 _AddConfig(lines, target, environment_plus, environment_minus,
1283                            environment_change)
1284
1285             _OutputConfigInfo(lines)
1286
1287         if show_config and self._base_config:
1288             summary = {}
1289             arch_config_plus = {}
1290             arch_config_minus = {}
1291             arch_config_change = {}
1292             arch_list = []
1293
1294             for target in board_dict:
1295                 if target not in board_selected:
1296                     continue
1297                 arch = board_selected[target].arch
1298                 if arch not in arch_list:
1299                     arch_list.append(arch)
1300
1301             for arch in arch_list:
1302                 arch_config_plus[arch] = {}
1303                 arch_config_minus[arch] = {}
1304                 arch_config_change[arch] = {}
1305                 for name in self.config_filenames:
1306                     arch_config_plus[arch][name] = {}
1307                     arch_config_minus[arch][name] = {}
1308                     arch_config_change[arch][name] = {}
1309
1310             for target in board_dict:
1311                 if target not in board_selected:
1312                     continue
1313
1314                 arch = board_selected[target].arch
1315
1316                 all_config_plus = {}
1317                 all_config_minus = {}
1318                 all_config_change = {}
1319                 tbase = self._base_config[target]
1320                 tconfig = config[target]
1321                 lines = []
1322                 for name in self.config_filenames:
1323                     if not tconfig.config[name]:
1324                         continue
1325                     config_plus = {}
1326                     config_minus = {}
1327                     config_change = {}
1328                     base = tbase.config[name]
1329                     for key, value in tconfig.config[name].iteritems():
1330                         if key not in base:
1331                             config_plus[key] = value
1332                             all_config_plus[key] = value
1333                     for key, value in base.iteritems():
1334                         if key not in tconfig.config[name]:
1335                             config_minus[key] = value
1336                             all_config_minus[key] = value
1337                     for key, value in base.iteritems():
1338                         new_value = tconfig.config.get(key)
1339                         if new_value and value != new_value:
1340                             desc = '%s -> %s' % (value, new_value)
1341                             config_change[key] = desc
1342                             all_config_change[key] = desc
1343
1344                     arch_config_plus[arch][name].update(config_plus)
1345                     arch_config_minus[arch][name].update(config_minus)
1346                     arch_config_change[arch][name].update(config_change)
1347
1348                     _AddConfig(lines, name, config_plus, config_minus,
1349                                config_change)
1350                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1351                            all_config_change)
1352                 summary[target] = '\n'.join(lines)
1353
1354             lines_by_target = {}
1355             for target, lines in summary.iteritems():
1356                 if lines in lines_by_target:
1357                     lines_by_target[lines].append(target)
1358                 else:
1359                     lines_by_target[lines] = [target]
1360
1361             for arch in arch_list:
1362                 lines = []
1363                 all_plus = {}
1364                 all_minus = {}
1365                 all_change = {}
1366                 for name in self.config_filenames:
1367                     all_plus.update(arch_config_plus[arch][name])
1368                     all_minus.update(arch_config_minus[arch][name])
1369                     all_change.update(arch_config_change[arch][name])
1370                     _AddConfig(lines, name, arch_config_plus[arch][name],
1371                                arch_config_minus[arch][name],
1372                                arch_config_change[arch][name])
1373                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1374                 #arch_summary[target] = '\n'.join(lines)
1375                 if lines:
1376                     Print('%s:' % arch)
1377                     _OutputConfigInfo(lines)
1378
1379             for lines, targets in lines_by_target.iteritems():
1380                 if not lines:
1381                     continue
1382                 Print('%s :' % ' '.join(sorted(targets)))
1383                 _OutputConfigInfo(lines.split('\n'))
1384
1385
1386         # Save our updated information for the next call to this function
1387         self._base_board_dict = board_dict
1388         self._base_err_lines = err_lines
1389         self._base_warn_lines = warn_lines
1390         self._base_err_line_boards = err_line_boards
1391         self._base_warn_line_boards = warn_line_boards
1392         self._base_config = config
1393         self._base_environment = environment
1394
1395         # Get a list of boards that did not get built, if needed
1396         not_built = []
1397         for board in board_selected:
1398             if not board in board_dict:
1399                 not_built.append(board)
1400         if not_built:
1401             Print("Boards not built (%d): %s" % (len(not_built),
1402                   ', '.join(not_built)))
1403
1404     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1405             (board_dict, err_lines, err_line_boards, warn_lines,
1406              warn_line_boards, config, environment) = self.GetResultSummary(
1407                     board_selected, commit_upto,
1408                     read_func_sizes=self._show_bloat,
1409                     read_config=self._show_config,
1410                     read_environment=self._show_environment)
1411             if commits:
1412                 msg = '%02d: %s' % (commit_upto + 1,
1413                         commits[commit_upto].subject)
1414                 Print(msg, colour=self.col.BLUE)
1415             self.PrintResultSummary(board_selected, board_dict,
1416                     err_lines if self._show_errors else [], err_line_boards,
1417                     warn_lines if self._show_errors else [], warn_line_boards,
1418                     config, environment, self._show_sizes, self._show_detail,
1419                     self._show_bloat, self._show_config, self._show_environment)
1420
1421     def ShowSummary(self, commits, board_selected):
1422         """Show a build summary for U-Boot for a given board list.
1423
1424         Reset the result summary, then repeatedly call GetResultSummary on
1425         each commit's results, then display the differences we see.
1426
1427         Args:
1428             commit: Commit objects to summarise
1429             board_selected: Dict containing boards to summarise
1430         """
1431         self.commit_count = len(commits) if commits else 1
1432         self.commits = commits
1433         self.ResetResultSummary(board_selected)
1434         self._error_lines = 0
1435
1436         for commit_upto in range(0, self.commit_count, self._step):
1437             self.ProduceResultSummary(commit_upto, commits, board_selected)
1438         if not self._error_lines:
1439             Print('(no errors to report)', colour=self.col.GREEN)
1440
1441
1442     def SetupBuild(self, board_selected, commits):
1443         """Set up ready to start a build.
1444
1445         Args:
1446             board_selected: Selected boards to build
1447             commits: Selected commits to build
1448         """
1449         # First work out how many commits we will build
1450         count = (self.commit_count + self._step - 1) / self._step
1451         self.count = len(board_selected) * count
1452         self.upto = self.warned = self.fail = 0
1453         self._timestamps = collections.deque()
1454
1455     def GetThreadDir(self, thread_num):
1456         """Get the directory path to the working dir for a thread.
1457
1458         Args:
1459             thread_num: Number of thread to check.
1460         """
1461         return os.path.join(self._working_dir, '%02d' % thread_num)
1462
1463     def _PrepareThread(self, thread_num, setup_git):
1464         """Prepare the working directory for a thread.
1465
1466         This clones or fetches the repo into the thread's work directory.
1467
1468         Args:
1469             thread_num: Thread number (0, 1, ...)
1470             setup_git: True to set up a git repo clone
1471         """
1472         thread_dir = self.GetThreadDir(thread_num)
1473         builderthread.Mkdir(thread_dir)
1474         git_dir = os.path.join(thread_dir, '.git')
1475
1476         # Clone the repo if it doesn't already exist
1477         # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1478         # we have a private index but uses the origin repo's contents?
1479         if setup_git and self.git_dir:
1480             src_dir = os.path.abspath(self.git_dir)
1481             if os.path.exists(git_dir):
1482                 gitutil.Fetch(git_dir, thread_dir)
1483             else:
1484                 Print('\rCloning repo for thread %d' % thread_num,
1485                       newline=False)
1486                 gitutil.Clone(src_dir, thread_dir)
1487                 Print('\r%s\r' % (' ' * 30), newline=False)
1488
1489     def _PrepareWorkingSpace(self, max_threads, setup_git):
1490         """Prepare the working directory for use.
1491
1492         Set up the git repo for each thread.
1493
1494         Args:
1495             max_threads: Maximum number of threads we expect to need.
1496             setup_git: True to set up a git repo clone
1497         """
1498         builderthread.Mkdir(self._working_dir)
1499         for thread in range(max_threads):
1500             self._PrepareThread(thread, setup_git)
1501
1502     def _PrepareOutputSpace(self):
1503         """Get the output directories ready to receive files.
1504
1505         We delete any output directories which look like ones we need to
1506         create. Having left over directories is confusing when the user wants
1507         to check the output manually.
1508         """
1509         if not self.commits:
1510             return
1511         dir_list = []
1512         for commit_upto in range(self.commit_count):
1513             dir_list.append(self._GetOutputDir(commit_upto))
1514
1515         to_remove = []
1516         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1517             if dirname not in dir_list:
1518                 to_remove.append(dirname)
1519         if to_remove:
1520             Print('Removing %d old build directories' % len(to_remove),
1521                   newline=False)
1522             for dirname in to_remove:
1523                 shutil.rmtree(dirname)
1524
1525     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1526         """Build all commits for a list of boards
1527
1528         Args:
1529             commits: List of commits to be build, each a Commit object
1530             boards_selected: Dict of selected boards, key is target name,
1531                     value is Board object
1532             keep_outputs: True to save build output files
1533             verbose: Display build results as they are completed
1534         Returns:
1535             Tuple containing:
1536                 - number of boards that failed to build
1537                 - number of boards that issued warnings
1538         """
1539         self.commit_count = len(commits) if commits else 1
1540         self.commits = commits
1541         self._verbose = verbose
1542
1543         self.ResetResultSummary(board_selected)
1544         builderthread.Mkdir(self.base_dir, parents = True)
1545         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1546                 commits is not None)
1547         self._PrepareOutputSpace()
1548         Print('\rStarting build...', newline=False)
1549         self.SetupBuild(board_selected, commits)
1550         self.ProcessResult(None)
1551
1552         # Create jobs to build all commits for each board
1553         for brd in board_selected.itervalues():
1554             job = builderthread.BuilderJob()
1555             job.board = brd
1556             job.commits = commits
1557             job.keep_outputs = keep_outputs
1558             job.step = self._step
1559             self.queue.put(job)
1560
1561         term = threading.Thread(target=self.queue.join)
1562         term.setDaemon(True)
1563         term.start()
1564         while term.isAlive():
1565             term.join(100)
1566
1567         # Wait until we have processed all output
1568         self.out_queue.join()
1569         Print()
1570         self.ClearLine(0)
1571         return (self.fail, self.warned)