X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=tools%2Fbuildman%2Fcontrol.py;h=73b1a14fb6bb8428206ffc4a9a8b8f4e0926b8ec;hb=a89c3a04bcb416102eaa0b7c398209dbc1c796a2;hp=4319ce77d31a8bccd247ea42b18d1112d3779d25;hpb=a19b0dd62d7b8efc658fa1aa685ff5665878f3ee;p=oweals%2Fu-boot.git diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 4319ce77d3..73b1a14fb6 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -1,26 +1,11 @@ # Copyright (c) 2013 The Chromium OS Authors. # -# See file CREDITS for list of people who contributed to this -# project. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of -# the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, -# MA 02111-1307 USA +# SPDX-License-Identifier: GPL-2.0+ # import multiprocessing import os +import shutil import sys import board @@ -29,21 +14,29 @@ from builder import Builder import gitutil import patchstream import terminal +from terminal import Print import toolchain +import command +import subprocess def GetPlural(count): """Returns a plural 's' if count is not 1""" return 's' if count != 1 else '' -def GetActionSummary(is_summary, count, selected, options): +def GetActionSummary(is_summary, commits, selected, options): """Return a string summarising the intended action. Returns: Summary string. """ - count = (count + options.step - 1) / options.step - str = '%s %d commit%s for %d boards' % ( - 'Summary of' if is_summary else 'Building', count, GetPlural(count), + if commits: + count = len(commits) + count = (count + options.step - 1) / options.step + commit_str = '%d commit%s' % (count, GetPlural(count)) + else: + commit_str = 'current source' + str = '%s %s for %d boards' % ( + 'Summary of' if is_summary else 'Building', commit_str, len(selected)) str += ' (%d thread%s, %d job%s per thread)' % (options.threads, GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) @@ -55,9 +48,9 @@ def ShowActions(series, why_selected, boards_selected, builder, options): Args: series: Series object why_selected: Dictionary where each key is a buildman argument - provided by the user, and the value is the boards brought - in by that argument. For example, 'arm' might bring in - 400 boards, so in this case the key would be 'arm' and + provided by the user, and the value is the list of boards + brought in by that argument. For example, 'arm' might bring + in 400 boards, so in this case the key would be 'arm' and the value would be a list of board names. boards_selected: Dict of selected boards, key is target name, value is Board object @@ -67,87 +60,173 @@ def ShowActions(series, why_selected, boards_selected, builder, options): col = terminal.Color() print 'Dry run, so not doing much. But I would do this:' print - print GetActionSummary(False, len(series.commits), boards_selected, + if series: + commits = series.commits + else: + commits = None + print GetActionSummary(False, commits, boards_selected, options) print 'Build directory: %s' % builder.base_dir - for upto in range(0, len(series.commits), options.step): - commit = series.commits[upto] - print ' ', col.Color(col.YELLOW, commit.hash, bright=False), - print commit.subject + if commits: + for upto in range(0, len(series.commits), options.step): + commit = series.commits[upto] + print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), + print commit.subject print for arg in why_selected: if arg != 'all': - print arg, ': %d boards' % why_selected[arg] + print arg, ': %d boards' % len(why_selected[arg]) + if options.verbose: + print ' %s' % ' '.join(why_selected[arg]) print ('Total boards to build for each commit: %d\n' % - why_selected['all']) + len(why_selected['all'])) -def DoBuildman(options, args): +def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, + clean_dir=False): """The main control code for buildman Args: options: Command line options object args: Command line arguments (list of strings) + toolchains: Toolchains to use - this should be a Toolchains() + object. If None, then it will be created and scanned + make_func: Make function to use for the builder. This is called + to execute 'make'. If this is None, the normal function + will be used, which calls the 'make' tool with suitable + arguments. This setting is useful for tests. + board: Boards() object to use, containing a list of available + boards. If this is None it will be created and scanned. """ + global builder + + if options.full_help: + pager = os.getenv('PAGER') + if not pager: + pager = 'more' + fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), + 'README') + command.Run(pager, fname) + return 0 + gitutil.Setup() + col = terminal.Color() - bsettings.Setup() options.git_dir = os.path.join(options.git, '.git') - toolchains = toolchain.Toolchains() - toolchains.Scan(options.list_tool_chains) + no_toolchains = toolchains is None + if no_toolchains: + toolchains = toolchain.Toolchains() + + if options.fetch_arch: + if options.fetch_arch == 'list': + sorted_list = toolchains.ListArchs() + print col.Color(col.BLUE, 'Available architectures: %s\n' % + ' '.join(sorted_list)) + return 0 + else: + fetch_arch = options.fetch_arch + if fetch_arch == 'all': + fetch_arch = ','.join(toolchains.ListArchs()) + print col.Color(col.CYAN, '\nDownloading toolchains: %s' % + fetch_arch) + for arch in fetch_arch.split(','): + print + ret = toolchains.FetchAndInstall(arch) + if ret: + return ret + return 0 + + if no_toolchains: + toolchains.GetSettings() + toolchains.Scan(options.list_tool_chains) if options.list_tool_chains: toolchains.List() print - return + return 0 # Work out how many commits to build. We want to build everything on the # branch. We also build the upstream commit as a control so we can see # problems introduced by the first commit on the branch. - col = terminal.Color() count = options.count + has_range = options.branch and '..' in options.branch if count == -1: if not options.branch: - str = 'Please use -b to specify a branch to build' - print col.Color(col.RED, str) - sys.exit(1) - count = gitutil.CountCommitsInBranch(options.git_dir, options.branch) - if count is None: - str = "Branch '%s' not found or has no upstream" % options.branch - print col.Color(col.RED, str) - sys.exit(1) - count += 1 # Build upstream commit also + count = 1 + else: + if has_range: + count, msg = gitutil.CountCommitsInRange(options.git_dir, + options.branch) + else: + count, msg = gitutil.CountCommitsInBranch(options.git_dir, + options.branch) + if count is None: + sys.exit(col.Color(col.RED, msg)) + elif count == 0: + sys.exit(col.Color(col.RED, "Range '%s' has no commits" % + options.branch)) + if msg: + print col.Color(col.YELLOW, msg) + count += 1 # Build upstream commit also if not count: str = ("No commits found to process in branch '%s': " "set branch's upstream or use -c flag" % options.branch) - print col.Color(col.RED, str) - sys.exit(1) + sys.exit(col.Color(col.RED, str)) # Work out what subset of the boards we are building - boards = board.Boards() - boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) - why_selected = boards.SelectBoards(args) + if not boards: + board_file = os.path.join(options.git, 'boards.cfg') + status = subprocess.call([os.path.join(options.git, + 'tools/genboardscfg.py')]) + if status != 0: + sys.exit("Failed to generate boards.cfg") + + boards = board.Boards() + boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) + + exclude = [] + if options.exclude: + for arg in options.exclude: + exclude += arg.split(',') + + why_selected = boards.SelectBoards(args, exclude) selected = boards.GetSelected() if not len(selected): - print col.Color(col.RED, 'No matching boards found') - sys.exit(1) + sys.exit(col.Color(col.RED, 'No matching boards found')) # Read the metadata from the commits. First look at the upstream commit, # then the ones in the branch. We would like to do something like # upstream/master~..branch but that isn't possible if upstream/master is # a merge commit (it will list all the commits that form part of the # merge) - range_expr = gitutil.GetRangeInBranch(options.git_dir, options.branch) - upstream_commit = gitutil.GetUpstream(options.git_dir, options.branch) - series = patchstream.GetMetaDataForList(upstream_commit, options.git_dir, - 1) # Conflicting tags are not a problem for buildman, since it does not use # them. For example, Series-version is not useful for buildman. On the # other hand conflicting tags will cause an error. So allow later tags - # to overwrite earlier ones. - series.allow_overwrite = True - series = patchstream.GetMetaDataForList(range_expr, options.git_dir, None, - series) + # to overwrite earlier ones by setting allow_overwrite=True + if options.branch: + if count == -1: + if has_range: + range_expr = options.branch + else: + range_expr = gitutil.GetRangeInBranch(options.git_dir, + options.branch) + upstream_commit = gitutil.GetUpstream(options.git_dir, + options.branch) + series = patchstream.GetMetaDataForList(upstream_commit, + options.git_dir, 1, series=None, allow_overwrite=True) + + series = patchstream.GetMetaDataForList(range_expr, + options.git_dir, None, series, allow_overwrite=True) + else: + # Honour the count + series = patchstream.GetMetaDataForList(options.branch, + options.git_dir, count, series=None, allow_overwrite=True) + else: + series = None + if not options.dry_run: + options.verbose = True + if not options.summary: + options.show_errors = True # By default we have one thread per CPU. But if there are not enough jobs # we can have fewer threads and use a high '-j' value for make. @@ -160,31 +239,72 @@ def DoBuildman(options, args): if not options.step: options.step = len(series.commits) - 1 - # Create a new builder with the selected options - output_dir = os.path.join('..', options.branch) + gnu_make = command.Output(os.path.join(options.git, + 'scripts/show-gnu-make'), raise_on_error=False).rstrip() + if not gnu_make: + sys.exit('GNU Make not found') + + # Create a new builder with the selected options. + output_dir = options.output_dir + if options.branch: + dirname = options.branch.replace('/', '_') + # As a special case allow the board directory to be placed in the + # output directory itself rather than any subdirectory. + if not options.no_subdirs: + output_dir = os.path.join(options.output_dir, dirname) + if (clean_dir and output_dir != options.output_dir and + os.path.exists(output_dir)): + shutil.rmtree(output_dir) builder = Builder(toolchains, output_dir, options.git_dir, - options.threads, options.jobs, checkout=True, - show_unknown=options.show_unknown, step=options.step) + options.threads, options.jobs, gnu_make=gnu_make, checkout=True, + show_unknown=options.show_unknown, step=options.step, + no_subdirs=options.no_subdirs, full_path=options.full_path, + verbose_build=options.verbose_build, + incremental=options.incremental, + per_board_out_dir=options.per_board_out_dir, + config_only=options.config_only, + squash_config_y=not options.preserve_config_y) builder.force_config_on_failure = not options.quick + if make_func: + builder.do_make = make_func # For a dry run, just show our actions as a sanity check if options.dry_run: ShowActions(series, why_selected, selected, builder, options) else: builder.force_build = options.force_build + builder.force_build_failures = options.force_build_failures + builder.force_reconfig = options.force_reconfig + builder.in_tree = options.in_tree # Work out which boards to build board_selected = boards.GetSelectedDict() - print GetActionSummary(options.summary, count, board_selected, options) + if series: + commits = series.commits + # Number the commits for test purposes + for commit in range(len(commits)): + commits[commit].sequence = commit + else: + commits = None + + Print(GetActionSummary(options.summary, commits, board_selected, + options)) + # We can't show function sizes without board details at present + if options.show_bloat: + options.show_detail = True + builder.SetDisplayOptions(options.show_errors, options.show_sizes, + options.show_detail, options.show_bloat, + options.list_error_boards, + options.show_config) if options.summary: - # We can't show function sizes without board details at present - if options.show_bloat: - options.show_detail = True - builder.ShowSummary(series.commits, board_selected, - options.show_errors, options.show_sizes, - options.show_detail, options.show_bloat) + builder.ShowSummary(commits, board_selected) else: - builder.BuildBoards(series.commits, board_selected, - options.show_errors, options.keep_outputs) + fail, warned = builder.BuildBoards(commits, board_selected, + options.keep_outputs, options.verbose) + if fail: + return 128 + elif warned: + return 129 + return 0