tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 from terminal import Print
  17 import toolchain
  18 import command
  19 import subprocess
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) // options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options,
  45                 board_warnings):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the list of boards
  52                 brought in by that argument. For example, 'arm' might bring
  53                 in 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59         board_warnings: List of warnings obtained from board selected
  60     """
  61     col = terminal.Color()
  62     print('Dry run, so not doing much. But I would do this:')
  63     print()
  64     if series:
  65         commits = series.commits
  66     else:
  67         commits = None
  68     print(GetActionSummary(False, commits, boards_selected,
  69             options))
  70     print('Build directory: %s' % builder.base_dir)
  71     if commits:
  72         for upto in range(0, len(series.commits), options.step):
  73             commit = series.commits[upto]
  74             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  75             print(commit.subject)
  76     print()
  77     for arg in why_selected:
  78         if arg != 'all':
  79             print(arg, ': %d boards' % len(why_selected[arg]))
  80             if options.verbose:
  81                 print('   %s' % ' '.join(why_selected[arg]))
  82     print(('Total boards to build for each commit: %d\n' %
  83             len(why_selected['all'])))
  84     if board_warnings:
  85         for warning in board_warnings:
  86             print(col.Color(col.YELLOW, warning))
  87
  88 def ShowToolchainPrefix(boards, toolchains):
  89     """Show information about a the tool chain used by one or more boards
  90
  91     The function checks that all boards use the same toolchain, then prints
  92     the correct value for CROSS_COMPILE.
  93
  94     Args:
  95         boards: Boards object containing selected boards
  96         toolchains: Toolchains object containing available toolchains
  97
  98     Return:
  99         None on success, string error message otherwise
 100     """
 101     boards = boards.GetSelectedDict()
 102     tc_set = set()
 103     for brd in boards.values():
 104         tc_set.add(toolchains.Select(brd.arch))
 105     if len(tc_set) != 1:
 106         return 'Supplied boards must share one toolchain'
 107         return False
 108     tc = tc_set.pop()
 109     print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 110     return None
 111
 112 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 113                clean_dir=False):
 114     """The main control code for buildman
 115
 116     Args:
 117         options: Command line options object
 118         args: Command line arguments (list of strings)
 119         toolchains: Toolchains to use - this should be a Toolchains()
 120                 object. If None, then it will be created and scanned
 121         make_func: Make function to use for the builder. This is called
 122                 to execute 'make'. If this is None, the normal function
 123                 will be used, which calls the 'make' tool with suitable
 124                 arguments. This setting is useful for tests.
 125         board: Boards() object to use, containing a list of available
 126                 boards. If this is None it will be created and scanned.
 127     """
 128     global builder
 129
 130     if options.full_help:
 131         pager = os.getenv('PAGER')
 132         if not pager:
 133             pager = 'more'
 134         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 135                              'README')
 136         command.Run(pager, fname)
 137         return 0
 138
 139     gitutil.Setup()
 140     col = terminal.Color()
 141
 142     options.git_dir = os.path.join(options.git, '.git')
 143
 144     no_toolchains = toolchains is None
 145     if no_toolchains:
 146         toolchains = toolchain.Toolchains(options.override_toolchain)
 147
 148     if options.fetch_arch:
 149         if options.fetch_arch == 'list':
 150             sorted_list = toolchains.ListArchs()
 151             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 152                             ' '.join(sorted_list)))
 153             return 0
 154         else:
 155             fetch_arch = options.fetch_arch
 156             if fetch_arch == 'all':
 157                 fetch_arch = ','.join(toolchains.ListArchs())
 158                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 159                                 fetch_arch))
 160             for arch in fetch_arch.split(','):
 161                 print()
 162                 ret = toolchains.FetchAndInstall(arch)
 163                 if ret:
 164                     return ret
 165             return 0
 166
 167     if no_toolchains:
 168         toolchains.GetSettings()
 169         toolchains.Scan(options.list_tool_chains and options.verbose)
 170     if options.list_tool_chains:
 171         toolchains.List()
 172         print()
 173         return 0
 174
 175     # Work out what subset of the boards we are building
 176     if not boards:
 177         if not os.path.exists(options.output_dir):
 178             os.makedirs(options.output_dir)
 179         board_file = os.path.join(options.output_dir, 'boards.cfg')
 180         genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 181         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 182         if status != 0:
 183             sys.exit("Failed to generate boards.cfg")
 184
 185         boards = board.Boards()
 186         boards.ReadBoards(board_file)
 187
 188     exclude = []
 189     if options.exclude:
 190         for arg in options.exclude:
 191             exclude += arg.split(',')
 192
 193     if options.boards:
 194         requested_boards = []
 195         for b in options.boards:
 196             requested_boards += b.split(',')
 197     else:
 198         requested_boards = None
 199     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 200                                                        requested_boards)
 201     selected = boards.GetSelected()
 202     if not len(selected):
 203         sys.exit(col.Color(col.RED, 'No matching boards found'))
 204
 205     if options.print_prefix:
 206         err = ShowToolchainInfo(boards, toolchains)
 207         if err:
 208             sys.exit(col.Color(col.RED, err))
 209         return 0
 210
 211     # Work out how many commits to build. We want to build everything on the
 212     # branch. We also build the upstream commit as a control so we can see
 213     # problems introduced by the first commit on the branch.
 214     count = options.count
 215     has_range = options.branch and '..' in options.branch
 216     if count == -1:
 217         if not options.branch:
 218             count = 1
 219         else:
 220             if has_range:
 221                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 222                                                          options.branch)
 223             else:
 224                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 225                                                           options.branch)
 226             if count is None:
 227                 sys.exit(col.Color(col.RED, msg))
 228             elif count == 0:
 229                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 230                                    options.branch))
 231             if msg:
 232                 print(col.Color(col.YELLOW, msg))
 233             count += 1   # Build upstream commit also
 234
 235     if not count:
 236         str = ("No commits found to process in branch '%s': "
 237                "set branch's upstream or use -c flag" % options.branch)
 238         sys.exit(col.Color(col.RED, str))
 239     if options.work_in_output:
 240         if len(selected) != 1:
 241             sys.exit(col.Color(col.RED,
 242                                '-w can only be used with a single board'))
 243         if count != 1:
 244             sys.exit(col.Color(col.RED,
 245                                '-w can only be used with a single commit'))
 246
 247     # Read the metadata from the commits. First look at the upstream commit,
 248     # then the ones in the branch. We would like to do something like
 249     # upstream/master~..branch but that isn't possible if upstream/master is
 250     # a merge commit (it will list all the commits that form part of the
 251     # merge)
 252     # Conflicting tags are not a problem for buildman, since it does not use
 253     # them. For example, Series-version is not useful for buildman. On the
 254     # other hand conflicting tags will cause an error. So allow later tags
 255     # to overwrite earlier ones by setting allow_overwrite=True
 256     if options.branch:
 257         if count == -1:
 258             if has_range:
 259                 range_expr = options.branch
 260             else:
 261                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 262                                                       options.branch)
 263             upstream_commit = gitutil.GetUpstream(options.git_dir,
 264                                                   options.branch)
 265             series = patchstream.GetMetaDataForList(upstream_commit,
 266                 options.git_dir, 1, series=None, allow_overwrite=True)
 267
 268             series = patchstream.GetMetaDataForList(range_expr,
 269                     options.git_dir, None, series, allow_overwrite=True)
 270         else:
 271             # Honour the count
 272             series = patchstream.GetMetaDataForList(options.branch,
 273                     options.git_dir, count, series=None, allow_overwrite=True)
 274     else:
 275         series = None
 276         if not options.dry_run:
 277             options.verbose = True
 278             if not options.summary:
 279                 options.show_errors = True
 280
 281     # By default we have one thread per CPU. But if there are not enough jobs
 282     # we can have fewer threads and use a high '-j' value for make.
 283     if not options.threads:
 284         options.threads = min(multiprocessing.cpu_count(), len(selected))
 285     if not options.jobs:
 286         options.jobs = max(1, (multiprocessing.cpu_count() +
 287                 len(selected) - 1) // len(selected))
 288
 289     if not options.step:
 290         options.step = len(series.commits) - 1
 291
 292     gnu_make = command.Output(os.path.join(options.git,
 293             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 294     if not gnu_make:
 295         sys.exit('GNU Make not found')
 296
 297     # Create a new builder with the selected options.
 298     output_dir = options.output_dir
 299     if options.branch:
 300         dirname = options.branch.replace('/', '_')
 301         # As a special case allow the board directory to be placed in the
 302         # output directory itself rather than any subdirectory.
 303         if not options.no_subdirs:
 304             output_dir = os.path.join(options.output_dir, dirname)
 305         if clean_dir and os.path.exists(output_dir):
 306             shutil.rmtree(output_dir)
 307     builder = Builder(toolchains, output_dir, options.git_dir,
 308             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 309             show_unknown=options.show_unknown, step=options.step,
 310             no_subdirs=options.no_subdirs, full_path=options.full_path,
 311             verbose_build=options.verbose_build,
 312             incremental=options.incremental,
 313             per_board_out_dir=options.per_board_out_dir,
 314             config_only=options.config_only,
 315             squash_config_y=not options.preserve_config_y,
 316             warnings_as_errors=options.warnings_as_errors,
 317             work_in_output=options.work_in_output)
 318     builder.force_config_on_failure = not options.quick
 319     if make_func:
 320         builder.do_make = make_func
 321
 322     # For a dry run, just show our actions as a sanity check
 323     if options.dry_run:
 324         ShowActions(series, why_selected, selected, builder, options,
 325                     board_warnings)
 326     else:
 327         builder.force_build = options.force_build
 328         builder.force_build_failures = options.force_build_failures
 329         builder.force_reconfig = options.force_reconfig
 330         builder.in_tree = options.in_tree
 331
 332         # Work out which boards to build
 333         board_selected = boards.GetSelectedDict()
 334
 335         if series:
 336             commits = series.commits
 337             # Number the commits for test purposes
 338             for commit in range(len(commits)):
 339                 commits[commit].sequence = commit
 340         else:
 341             commits = None
 342
 343         Print(GetActionSummary(options.summary, commits, board_selected,
 344                                 options))
 345
 346         # We can't show function sizes without board details at present
 347         if options.show_bloat:
 348             options.show_detail = True
 349         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 350                                   options.show_detail, options.show_bloat,
 351                                   options.list_error_boards,
 352                                   options.show_config,
 353                                   options.show_environment)
 354         if options.summary:
 355             builder.ShowSummary(commits, board_selected)
 356         else:
 357             fail, warned = builder.BuildBoards(commits, board_selected,
 358                                 options.keep_outputs, options.verbose)
 359             if fail:
 360                 return 128
 361             elif warned and not options.ignore_warnings:
 362                 return 129
 363     return 0