X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;ds=sidebyside;f=tools%2Fbuildman%2Fbuilder.py;h=54f3292208a9f0ccf64a71758b2040f912880fce;hb=073adf987e8251ad934fcac4fd1bf20d4f34f96e;hp=e2da0eb299299b01daf1eac793e77347288b46d3;hpb=190064b4daa30ce8efa7bcbee12ac05b780fb9c8;p=oweals%2Fu-boot.git diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index e2da0eb299..54f3292208 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -20,6 +20,7 @@ import builderthread import command import gitutil import terminal +from terminal import Print import toolchain @@ -140,6 +141,7 @@ class Builder: Private members: _base_board_dict: Last-summarised Dict of boards _base_err_lines: Last-summarised list of errors + _base_warn_lines: Last-summarised list of warnings _build_period_us: Time taken for a single build (float object). _complete_delay: Expected delay until completion (timedelta) _next_delay_update: Next time we plan to display a progress update @@ -172,7 +174,8 @@ class Builder: self.func_sizes = func_sizes def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, - gnu_make='make', checkout=True, show_unknown=True, step=1): + gnu_make='make', checkout=True, show_unknown=True, step=1, + no_subdirs=False, full_path=False, verbose_build=False): """Create a new Builder object Args: @@ -186,6 +189,11 @@ class Builder: This is used for testing. show_unknown: Show unknown boards (those not built) in summary step: 1 to process every commit, n to process every nth commit + no_subdirs: Don't create subdirectories when building current + source for a single board + full_path: Return the full path in CROSS_COMPILE and don't set + PATH + verbose_build: Run build with V=1 and don't use 'make -s' """ self.toolchains = toolchains self.base_dir = base_dir @@ -210,9 +218,18 @@ class Builder: self.force_reconfig = False self._step = step self.in_tree = False + self._error_lines = 0 + self.no_subdirs = no_subdirs + self.full_path = full_path + self.verbose_build = verbose_build self.col = terminal.Color() + self._re_function = re.compile('(.*): In function.*') + self._re_files = re.compile('In file included from.*') + self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') + self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') + self.queue = Queue.Queue() self.out_queue = Queue.Queue() for i in range(self.num_threads): @@ -235,6 +252,23 @@ class Builder: for t in self.threads: del t + def SetDisplayOptions(self, show_errors=False, show_sizes=False, + show_detail=False, show_bloat=False, + list_error_boards=False): + """Setup display options for the builder. + + show_errors: True to show summarised error/warning info + show_sizes: Show size deltas + show_detail: Show detail for each board + show_bloat: Show detail for each function + list_error_boards: Show the boards which caused each error/warning + """ + self._show_errors = show_errors + self._show_sizes = show_sizes + self._show_detail = show_detail + self._show_bloat = show_bloat + self._list_error_boards = list_error_boards + def _AddTimestamp(self): """Add a new timestamp to the list and record the build period. @@ -275,8 +309,8 @@ class Builder: length: Length of new line, in characters """ if length < self.last_line_len: - print ' ' * (self.last_line_len - length), - print '\r', + Print(' ' * (self.last_line_len - length), newline=False) + Print('\r', newline=False) self.last_line_len = length sys.stdout.flush() @@ -293,7 +327,7 @@ class Builder: Args: commit: Commit object that is being built brd: Board object that is being built - stage: Stage that we are at (distclean, config, build) + stage: Stage that we are at (mrproper, config, build) cwd: Directory where make should be run args: Arguments to pass to make kwargs: Arguments to pass to command.RunPipe() @@ -307,7 +341,8 @@ class Builder: """Process the result of a build, showing progress information Args: - result: A CommandResult object + result: A CommandResult object, which indicates the result for + a single build """ col = terminal.Color() if result: @@ -325,6 +360,13 @@ class Builder: self.warned += 1 if result.already_done: self.already_done += 1 + if self._verbose: + Print('\r', newline=False) + self.ClearLine(0) + boards_selected = {target : result.brd} + self.ResetResultSummary(boards_selected) + self.ProduceResultSummary(result.commit_upto, self.commits, + boards_selected) else: target = '(starting)' @@ -347,8 +389,8 @@ class Builder: self.commit_count) name += target - print line + name, - length = 13 + len(name) + Print(line + name, newline=False) + length = 14 + len(name) self.ClearLine(length) def _GetOutputDir(self, commit_upto): @@ -359,15 +401,17 @@ class Builder: Args: commit_upto: Commit number to use (0..self.count-1) """ + commit_dir = None if self.commits: commit = self.commits[commit_upto] subject = commit.subject.translate(trans_valid_chars) commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, self.commit_count, commit.hash, subject[:20])) - else: + elif not self.no_subdirs: commit_dir = 'current' - output_dir = os.path.join(self.base_dir, commit_dir) - return output_dir + if not commit_dir: + return self.base_dir + return os.path.join(self.base_dir, commit_dir) def GetBuildDir(self, commit_upto, target): """Get the name of the build directory for a commit number @@ -463,7 +507,7 @@ class Builder: try: size, type, name = line[:-1].split() except: - print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) + Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) continue if type in 'tTdDbB': # function names begin with '.' on 64-bit powerpc @@ -546,19 +590,57 @@ class Builder: Tuple: Dict containing boards which passed building this commit. keyed by board.target - List containing a summary of error/warning lines + List containing a summary of error lines + Dict keyed by error line, containing a list of the Board + objects with that error + List containing a summary of warning lines + Dict keyed by error line, containing a list of the Board + objects with that warning """ + def AddLine(lines_summary, lines_boards, line, board): + line = line.rstrip() + if line in lines_boards: + lines_boards[line].append(board) + else: + lines_boards[line] = [board] + lines_summary.append(line) + board_dict = {} err_lines_summary = [] + err_lines_boards = {} + warn_lines_summary = [] + warn_lines_boards = {} for board in boards_selected.itervalues(): outcome = self.GetBuildOutcome(commit_upto, board.target, read_func_sizes) board_dict[board.target] = outcome - for err in outcome.err_lines: - if err and not err.rstrip() in err_lines_summary: - err_lines_summary.append(err.rstrip()) - return board_dict, err_lines_summary + last_func = None + last_was_warning = False + for line in outcome.err_lines: + if line: + if (self._re_function.match(line) or + self._re_files.match(line)): + last_func = line + else: + is_warning = self._re_warning.match(line) + is_note = self._re_note.match(line) + if is_warning or (last_was_warning and is_note): + if last_func: + AddLine(warn_lines_summary, warn_lines_boards, + last_func, board) + AddLine(warn_lines_summary, warn_lines_boards, + line, board) + else: + if last_func: + AddLine(err_lines_summary, err_lines_boards, + last_func, board) + AddLine(err_lines_summary, err_lines_boards, + line, board) + last_was_warning = is_warning + last_func = None + return (board_dict, err_lines_summary, err_lines_boards, + warn_lines_summary, warn_lines_boards) def AddOutcome(self, board_dict, arch_list, changes, char, color): """Add an output to our list of outcomes for each architecture @@ -582,7 +664,7 @@ class Builder: arch = 'unknown' str = self.col.Color(color, ' ' + target) if not arch in done_arch: - str = self.col.Color(color, char) + ' ' + str + str = ' %s %s' % (self.col.Color(color, char), str) done_arch[arch] = True if not arch in arch_list: arch_list[arch] = str @@ -613,6 +695,9 @@ class Builder: for board in board_selected: self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) self._base_err_lines = [] + self._base_warn_lines = [] + self._base_err_line_boards = {} + self._base_warn_line_boards = {} def PrintFuncSizeDetail(self, fname, old, new): grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 @@ -650,16 +735,16 @@ class Builder: return args = [self.ColourNum(x) for x in args] indent = ' ' * 15 - print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % - tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) - print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', - 'delta') + Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % + tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) + Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', + 'delta')) for diff, name in delta: if diff: color = self.col.RED if diff > 0 else self.col.GREEN msg = '%s %-38s %7s %7s %+7d' % (indent, name, old.get(name, '-'), new.get(name,'-'), diff) - print self.col.Color(color, msg) + Print(msg, colour=color) def PrintSizeDetail(self, target_list, show_bloat): @@ -684,11 +769,12 @@ class Builder: color = self.col.RED if diff > 0 else self.col.GREEN msg = ' %s %+d' % (name, diff) if not printed_target: - print '%10s %-15s:' % ('', result['_target']), + Print('%10s %-15s:' % ('', result['_target']), + newline=False) printed_target = True - print self.col.Color(color, msg), + Print(msg, colour=color, newline=False) if printed_target: - print + Print() if show_bloat: target = result['_target'] outcome = result['_outcome'] @@ -793,18 +879,19 @@ class Builder: color = self.col.RED if avg_diff > 0 else self.col.GREEN msg = ' %s %+1.1f' % (name, avg_diff) if not printed_arch: - print '%10s: (for %d/%d boards)' % (arch, count, - arch_count[arch]), + Print('%10s: (for %d/%d boards)' % (arch, count, + arch_count[arch]), newline=False) printed_arch = True - print self.col.Color(color, msg), + Print(msg, colour=color, newline=False) if printed_arch: - print + Print() if show_detail: self.PrintSizeDetail(target_list, show_bloat) def PrintResultSummary(self, board_selected, board_dict, err_lines, + err_line_boards, warn_lines, warn_line_boards, show_sizes, show_detail, show_bloat): """Compare results with the base results and display delta. @@ -820,10 +907,49 @@ class Builder: commit, keyed by board.target. The value is an Outcome object. err_lines: A list of errors for this commit, or [] if there is none, or we don't want to print errors + err_line_boards: Dict keyed by error line, containing a list of + the Board objects with that error + warn_lines: A list of warnings for this commit, or [] if there is + none, or we don't want to print errors + warn_line_boards: Dict keyed by warning line, containing a list of + the Board objects with that warning show_sizes: Show image size deltas show_detail: Show detail for each board show_bloat: Show detail for each function """ + def _BoardList(line, line_boards): + """Helper function to get a line of boards containing a line + + Args: + line: Error line to search for + Return: + String containing a list of boards with that error line, or + '' if the user has not requested such a list + """ + if self._list_error_boards: + names = [] + for board in line_boards[line]: + if not board.target in names: + names.append(board.target) + names_str = '(%s) ' % ','.join(names) + else: + names_str = '' + return names_str + + def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, + char): + better_lines = [] + worse_lines = [] + for line in lines: + if line not in base_lines: + worse_lines.append(char + '+' + + _BoardList(line, line_boards) + line) + for line in base_lines: + if line not in lines: + better_lines.append(char + '-' + + _BoardList(line, base_line_boards) + line) + return better_lines, worse_lines + better = [] # List of boards fixed since last commit worse = [] # List of new broken boards since last commit new = [] # List of boards that didn't exist last time @@ -847,17 +973,14 @@ class Builder: new.append(target) # Get a list of errors that have appeared, and disappeared - better_err = [] - worse_err = [] - for line in err_lines: - if line not in self._base_err_lines: - worse_err.append('+' + line) - for line in self._base_err_lines: - if line not in err_lines: - better_err.append('-' + line) + better_err, worse_err = _CalcErrorDelta(self._base_err_lines, + self._base_err_line_boards, err_lines, err_line_boards, '') + better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, + self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') # Display results by arch - if better or worse or unknown or new or worse_err or better_err: + if (better or worse or unknown or new or worse_err or better_err + or worse_warn or better_warn): arch_list = {} self.AddOutcome(board_selected, arch_list, better, '', self.col.GREEN) @@ -868,11 +991,20 @@ class Builder: self.AddOutcome(board_selected, arch_list, unknown, '?', self.col.MAGENTA) for arch, target_list in arch_list.iteritems(): - print '%10s: %s' % (arch, target_list) + Print('%10s: %s' % (arch, target_list)) + self._error_lines += 1 if better_err: - print self.col.Color(self.col.GREEN, '\n'.join(better_err)) + Print('\n'.join(better_err), colour=self.col.GREEN) + self._error_lines += 1 if worse_err: - print self.col.Color(self.col.RED, '\n'.join(worse_err)) + Print('\n'.join(worse_err), colour=self.col.RED) + self._error_lines += 1 + if better_warn: + Print('\n'.join(better_warn), colour=self.col.CYAN) + self._error_lines += 1 + if worse_warn: + Print('\n'.join(worse_warn), colour=self.col.MAGENTA) + self._error_lines += 1 if show_sizes: self.PrintSizeSummary(board_selected, board_dict, show_detail, @@ -881,6 +1013,9 @@ class Builder: # Save our updated information for the next call to this function self._base_board_dict = board_dict self._base_err_lines = err_lines + self._base_warn_lines = warn_lines + self._base_err_line_boards = err_line_boards + self._base_warn_line_boards = warn_line_boards # Get a list of boards that did not get built, if needed not_built = [] @@ -888,12 +1023,24 @@ class Builder: if not board in board_dict: not_built.append(board) if not_built: - print "Boards not built (%d): %s" % (len(not_built), - ', '.join(not_built)) - + Print("Boards not built (%d): %s" % (len(not_built), + ', '.join(not_built))) + + def ProduceResultSummary(self, commit_upto, commits, board_selected): + (board_dict, err_lines, err_line_boards, warn_lines, + warn_line_boards) = self.GetResultSummary( + board_selected, commit_upto, + read_func_sizes=self._show_bloat) + if commits: + msg = '%02d: %s' % (commit_upto + 1, + commits[commit_upto].subject) + Print(msg, colour=self.col.BLUE) + self.PrintResultSummary(board_selected, board_dict, + err_lines if self._show_errors else [], err_line_boards, + warn_lines if self._show_errors else [], warn_line_boards, + self._show_sizes, self._show_detail, self._show_bloat) - def ShowSummary(self, commits, board_selected, show_errors, show_sizes, - show_detail, show_bloat): + def ShowSummary(self, commits, board_selected): """Show a build summary for U-Boot for a given board list. Reset the result summary, then repeatedly call GetResultSummary on @@ -902,27 +1049,16 @@ class Builder: Args: commit: Commit objects to summarise board_selected: Dict containing boards to summarise - show_errors: Show errors that occured - show_sizes: Show size deltas - show_detail: Show detail for each board - show_bloat: Show detail for each function """ self.commit_count = len(commits) if commits else 1 self.commits = commits self.ResetResultSummary(board_selected) + self._error_lines = 0 for commit_upto in range(0, self.commit_count, self._step): - board_dict, err_lines = self.GetResultSummary(board_selected, - commit_upto, read_func_sizes=show_bloat) - if commits: - msg = '%02d: %s' % (commit_upto + 1, - commits[commit_upto].subject) - else: - msg = 'current' - print self.col.Color(self.col.BLUE, msg) - self.PrintResultSummary(board_selected, board_dict, - err_lines if show_errors else [], show_sizes, show_detail, - show_bloat) + self.ProduceResultSummary(commit_upto, commits, board_selected) + if not self._error_lines: + Print('(no errors to report)', colour=self.col.GREEN) def SetupBuild(self, board_selected, commits): @@ -938,40 +1074,6 @@ class Builder: self.upto = self.warned = self.fail = 0 self._timestamps = collections.deque() - def BuildBoardsForCommit(self, board_selected, keep_outputs): - """Build all boards for a single commit""" - self.SetupBuild(board_selected) - self.count = len(board_selected) - for brd in board_selected.itervalues(): - job = BuilderJob() - job.board = brd - job.commits = None - job.keep_outputs = keep_outputs - self.queue.put(brd) - - self.queue.join() - self.out_queue.join() - print - self.ClearLine(0) - - def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): - """Build all boards for all commits (non-incremental)""" - self.commit_count = len(commits) - - self.ResetResultSummary(board_selected) - for self.commit_upto in range(self.commit_count): - self.SelectCommit(commits[self.commit_upto]) - self.SelectOutputDir() - builderthread.Mkdir(self.output_dir) - - self.BuildBoardsForCommit(board_selected, keep_outputs) - board_dict, err_lines = self.GetResultSummary() - self.PrintResultSummary(board_selected, board_dict, - err_lines if show_errors else []) - - if self.already_done: - print '%d builds already done' % self.already_done - def GetThreadDir(self, thread_num): """Get the directory path to the working dir for a thread. @@ -1001,7 +1103,7 @@ class Builder: if os.path.exists(git_dir): gitutil.Fetch(git_dir, thread_dir) else: - print 'Cloning repo for thread %d' % thread_num + Print('Cloning repo for thread %d' % thread_num) gitutil.Clone(src_dir, thread_dir) def _PrepareWorkingSpace(self, max_threads, setup_git): @@ -1024,6 +1126,8 @@ class Builder: create. Having left over directories is confusing when the user wants to check the output manually. """ + if not self.commits: + return dir_list = [] for commit_upto in range(self.commit_count): dir_list.append(self._GetOutputDir(commit_upto)) @@ -1032,21 +1136,26 @@ class Builder: if dirname not in dir_list: shutil.rmtree(dirname) - def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): + def BuildBoards(self, commits, board_selected, keep_outputs, verbose): """Build all commits for a list of boards Args: commits: List of commits to be build, each a Commit object boards_selected: Dict of selected boards, key is target name, value is Board object - show_errors: True to show summarised error/warning info keep_outputs: True to save build output files + verbose: Display build results as they are completed + Returns: + Tuple containing: + - number of boards that failed to build + - number of boards that issued warnings """ self.commit_count = len(commits) if commits else 1 self.commits = commits + self._verbose = verbose self.ResetResultSummary(board_selected) - builderthread.Mkdir(self.base_dir) + builderthread.Mkdir(self.base_dir, parents = True) self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), commits is not None) self._PrepareOutputSpace() @@ -1067,5 +1176,6 @@ class Builder: # Wait until we have processed all output self.out_queue.join() - print + Print() self.ClearLine(0) + return (self.fail, self.warned)