from terminal import Print
import toolchain
-
"""
Theory of Operation
.git/ repository
"""
+"""Holds information about a particular error line we are outputing
+
+ char: Character representation: '+': error, '-': fixed error, 'w+': warning,
+ 'w-' = fixed warning
+ boards: List of Board objects which have line in the error/warning output
+ errline: The text of the error line
+"""
+ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
+
# Possible build outcomes
OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
force_build_failures: If a previously-built build (i.e. built on
a previous run of buildman) is marked as failed, rebuild it.
git_dir: Git directory containing source repository
- last_line_len: Length of the last line we printed (used for erasing
- it with new progress information)
num_jobs: Number of jobs to run at once (passed to make as -j)
num_threads: Number of builder threads to run
out_queue: Queue of results to process
_next_delay_update: Next time we plan to display a progress update
(datatime)
_show_unknown: Show unknown boards (those not built) in summary
+ _start_time: Start time for the build
_timestamps: List of timestamps for the completion of the last
last _timestamp_count builds. Each is a datetime object.
_timestamp_count: Number of timestamps to keep in our list.
def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
gnu_make='make', checkout=True, show_unknown=True, step=1,
no_subdirs=False, full_path=False, verbose_build=False,
- incremental=False, per_board_out_dir=False,
+ mrproper=False, per_board_out_dir=False,
config_only=False, squash_config_y=False,
warnings_as_errors=False, work_in_output=False):
"""Create a new Builder object
full_path: Return the full path in CROSS_COMPILE and don't set
PATH
verbose_build: Run build with V=1 and don't use 'make -s'
- incremental: Always perform incremental builds; don't run make
- mrproper when configuring
+ mrproper: Always run 'make mrproper' when configuring
per_board_out_dir: Build in a separate persistent directory per
board rather than a thread-specific directory
config_only: Only configure each build, don't build it
self._build_period_us = None
self._complete_delay = None
self._next_delay_update = datetime.now()
+ self._start_time = datetime.now()
self.force_config_on_failure = True
self.force_build_failures = False
self.force_reconfig = False
self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
self._re_dtb_warning = re.compile('(.*): Warning .*')
self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
+ self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
+ re.MULTILINE | re.DOTALL)
self.queue = queue.Queue()
self.out_queue = queue.Queue()
for i in range(self.num_threads):
- t = builderthread.BuilderThread(self, i, incremental,
+ t = builderthread.BuilderThread(self, i, mrproper,
per_board_out_dir)
t.setDaemon(True)
t.start()
self.threads.append(t)
- self.last_line_len = 0
t = builderthread.ResultThread(self)
t.setDaemon(True)
t.start()
def SetDisplayOptions(self, show_errors=False, show_sizes=False,
show_detail=False, show_bloat=False,
list_error_boards=False, show_config=False,
- show_environment=False):
+ show_environment=False, filter_dtb_warnings=False,
+ filter_migration_warnings=False):
"""Setup display options for the builder.
- show_errors: True to show summarised error/warning info
- show_sizes: Show size deltas
- show_detail: Show size delta detail for each board if show_sizes
- show_bloat: Show detail for each function
- list_error_boards: Show the boards which caused each error/warning
- show_config: Show config deltas
- show_environment: Show environment deltas
+ Args:
+ show_errors: True to show summarised error/warning info
+ show_sizes: Show size deltas
+ show_detail: Show size delta detail for each board if show_sizes
+ show_bloat: Show detail for each function
+ list_error_boards: Show the boards which caused each error/warning
+ show_config: Show config deltas
+ show_environment: Show environment deltas
+ filter_dtb_warnings: Filter out any warnings from the device-tree
+ compiler
+ filter_migration_warnings: Filter out any warnings about migrating
+ a board to driver model
"""
self._show_errors = show_errors
self._show_sizes = show_sizes
self._list_error_boards = list_error_boards
self._show_config = show_config
self._show_environment = show_environment
+ self._filter_dtb_warnings = filter_dtb_warnings
+ self._filter_migration_warnings = filter_migration_warnings
def _AddTimestamp(self):
"""Add a new timestamp to the list and record the build period.
self._timestamps.popleft()
count -= 1
- def ClearLine(self, length):
- """Clear any characters on the current line
-
- Make way for a new line of length 'length', by outputting enough
- spaces to clear out the old line. Then remember the new length for
- next time.
-
- Args:
- length: Length of new line, in characters
- """
- if length < self.last_line_len:
- Print(' ' * (self.last_line_len - length), newline=False)
- Print('\r', newline=False)
- self.last_line_len = length
- sys.stdout.flush()
-
def SelectCommit(self, commit, checkout=True):
"""Checkout the selected commit for this build
"""
if result.already_done:
self.already_done += 1
if self._verbose:
- Print('\r', newline=False)
- self.ClearLine(0)
+ terminal.PrintClear()
boards_selected = {target : result.brd}
self.ResetResultSummary(boards_selected)
self.ProduceResultSummary(result.commit_upto, self.commits,
line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
line += self.col.Color(self.col.RED, '%5d' % self.fail)
- name = ' /%-5d ' % self.count
+ line += ' /%-5d ' % self.count
+ remaining = self.count - self.upto
+ if remaining:
+ line += self.col.Color(self.col.MAGENTA, ' -%-5d ' % remaining)
+ else:
+ line += ' ' * 8
# Add our current completion time estimate
self._AddTimestamp()
if self._complete_delay:
- name += '%s : ' % self._complete_delay
- # When building all boards for a commit, we can print a commit
- # progress message.
- if result and result.commit_upto is None:
- name += 'commit %2d/%-3d' % (self.commit_upto + 1,
- self.commit_count)
-
- name += target
- Print(line + name, newline=False)
- length = 16 + len(name)
- self.ClearLine(length)
+ line += '%s : ' % self._complete_delay
+
+ line += target
+ terminal.PrintClear()
+ Print(line, newline=False, limit_to_line=True)
def _GetOutputDir(self, commit_upto):
"""Get the name of the output directory for a commit number
New list with only interesting lines included
"""
out_lines = []
+ if self._filter_migration_warnings:
+ text = '\n'.join(lines)
+ text = self._re_migration_warning.sub('', text)
+ lines = text.splitlines()
for line in lines:
- if not self.re_make_err.search(line):
- out_lines.append(line)
+ if self.re_make_err.search(line):
+ continue
+ if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
+ continue
+ out_lines.append(line)
return out_lines
def ReadFuncSizes(self, fname, fd):
Args:
line: Error line to search for
+ line_boards: boards to search, each a Board
Return:
- String containing a list of boards with that error line, or
- '' if the user has not requested such a list
+ List of boards with that error line, or [] if the user has not
+ requested such a list
"""
+ boards = []
+ board_set = set()
if self._list_error_boards:
- names = []
for board in line_boards[line]:
- if not board.target in names:
- names.append(board.target)
- names_str = '(%s) ' % ','.join(names)
- else:
- names_str = ''
- return names_str
+ if not board in board_set:
+ boards.append(board)
+ board_set.add(board)
+ return boards
def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
char):
+ """Calculate the required output based on changes in errors
+
+ Args:
+ base_lines: List of errors/warnings for previous commit
+ base_line_boards: Dict keyed by error line, containing a list
+ of the Board objects with that error in the previous commit
+ lines: List of errors/warning for this commit, each a str
+ line_boards: Dict keyed by error line, containing a list
+ of the Board objects with that error in this commit
+ char: Character representing error ('') or warning ('w'). The
+ broken ('+') or fixed ('-') characters are added in this
+ function
+
+ Returns:
+ Tuple
+ List of ErrLine objects for 'better' lines
+ List of ErrLine objects for 'worse' lines
+ """
better_lines = []
worse_lines = []
for line in lines:
if line not in base_lines:
- worse_lines.append(char + '+' +
- _BoardList(line, line_boards) + line)
+ errline = ErrLine(char + '+', _BoardList(line, line_boards),
+ line)
+ worse_lines.append(errline)
for line in base_lines:
if line not in lines:
- better_lines.append(char + '-' +
- _BoardList(line, base_line_boards) + line)
+ errline = ErrLine(char + '-',
+ _BoardList(line, base_line_boards), line)
+ better_lines.append(errline)
return better_lines, worse_lines
def _CalcConfig(delta, name, config):
col = self.col.YELLOW
Print(' ' + line, newline=True, colour=col)
+ def _OutputErrLines(err_lines, colour):
+ """Output the line of error/warning lines, if not empty
+
+ Also increments self._error_lines if err_lines not empty
+
+ Args:
+ err_lines: List of ErrLine objects, each an error or warning
+ line, possibly including a list of boards with that
+ error/warning
+ colour: Colour to use for output
+ """
+ if err_lines:
+ out_list = []
+ for line in err_lines:
+ boards = ''
+ names = [board.target for board in line.boards]
+ board_str = ' '.join(names) if names else ''
+ if board_str:
+ out = self.col.Color(colour, line.char + '(')
+ out += self.col.Color(self.col.MAGENTA, board_str,
+ bright=False)
+ out += self.col.Color(colour, ') %s' % line.errline)
+ else:
+ out = self.col.Color(colour, line.char + line.errline)
+ out_list.append(out)
+ Print('\n'.join(out_list))
+ self._error_lines += 1
+
ok_boards = [] # List of boards fixed since last commit
warn_boards = [] # List of boards with warnings since last commit
else:
new_boards.append(target)
- # Get a list of errors that have appeared, and disappeared
+ # Get a list of errors and warnings that have appeared, and disappeared
better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
self._base_err_line_boards, err_lines, err_line_boards, '')
better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
for arch, target_list in arch_list.items():
Print('%10s: %s' % (arch, target_list))
self._error_lines += 1
- if better_err:
- Print('\n'.join(better_err), colour=self.col.GREEN)
- self._error_lines += 1
- if worse_err:
- Print('\n'.join(worse_err), colour=self.col.RED)
- self._error_lines += 1
- if better_warn:
- Print('\n'.join(better_warn), colour=self.col.CYAN)
- self._error_lines += 1
- if worse_warn:
- Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
- self._error_lines += 1
+ _OutputErrLines(better_err, colour=self.col.GREEN)
+ _OutputErrLines(worse_err, colour=self.col.RED)
+ _OutputErrLines(better_warn, colour=self.col.CYAN)
+ _OutputErrLines(worse_warn, colour=self.col.YELLOW)
if show_sizes:
self.PrintSizeSummary(board_selected, board_dict, show_detail,
if setup_git and self.git_dir:
src_dir = os.path.abspath(self.git_dir)
if os.path.exists(git_dir):
+ Print('\rFetching repo for thread %d' % thread_num,
+ newline=False)
gitutil.Fetch(git_dir, thread_dir)
+ terminal.PrintClear()
else:
Print('\rCloning repo for thread %d' % thread_num,
newline=False)
gitutil.Clone(src_dir, thread_dir)
- Print('\r%s\r' % (' ' * 30), newline=False)
+ terminal.PrintClear()
def _PrepareWorkingSpace(self, max_threads, setup_git):
"""Prepare the working directory for use.
"""
to_remove = self._GetOutputSpaceRemovals()
if to_remove:
- Print('Removing %d old build directories' % len(to_remove),
+ Print('Removing %d old build directories...' % len(to_remove),
newline=False)
for dirname in to_remove:
shutil.rmtree(dirname)
+ terminal.PrintClear()
def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
"""Build all commits for a list of boards
# Wait until we have processed all output
self.out_queue.join()
Print()
- self.ClearLine(0)
+
+ msg = 'Completed: %d total built' % self.count
+ if self.already_done:
+ msg += ' (%d previously' % self.already_done
+ if self.already_done != self.count:
+ msg += ', %d newly' % (self.count - self.already_done)
+ msg += ')'
+ duration = datetime.now() - self._start_time
+ if duration > timedelta(microseconds=1000000):
+ if duration.microseconds >= 500000:
+ duration = duration + timedelta(seconds=1)
+ duration = duration - timedelta(microseconds=duration.microseconds)
+ msg += ', duration %s' % duration
+ Print(msg)
+
return (self.fail, self.warned)
projects / oweals / u-boot.git / blobdiff