tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 from terminal import Print
  17 import toolchain
  18 import command
  19 import subprocess
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) // options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options,
  45                 board_warnings):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the list of boards
  52                 brought in by that argument. For example, 'arm' might bring
  53                 in 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59         board_warnings: List of warnings obtained from board selected
  60     """
  61     col = terminal.Color()
  62     print('Dry run, so not doing much. But I would do this:')
  63     print()
  64     if series:
  65         commits = series.commits
  66     else:
  67         commits = None
  68     print(GetActionSummary(False, commits, boards_selected,
  69             options))
  70     print('Build directory: %s' % builder.base_dir)
  71     if commits:
  72         for upto in range(0, len(series.commits), options.step):
  73             commit = series.commits[upto]
  74             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  75             print(commit.subject)
  76     print()
  77     for arg in why_selected:
  78         if arg != 'all':
  79             print(arg, ': %d boards' % len(why_selected[arg]))
  80             if options.verbose:
  81                 print('   %s' % ' '.join(why_selected[arg]))
  82     print(('Total boards to build for each commit: %d\n' %
  83             len(why_selected['all'])))
  84     if board_warnings:
  85         for warning in board_warnings:
  86             print(col.Color(col.YELLOW, warning))
  87
  88 def CheckOutputDir(output_dir):
  89     """Make sure that the output directory is not within the current directory
  90
  91     If we try to use an output directory which is within the current directory
  92     (which is assumed to hold the U-Boot source) we may end up deleting the
  93     U-Boot source code. Detect this and print an error in this case.
  94
  95     Args:
  96         output_dir: Output directory path to check
  97     """
  98     path = os.path.realpath(output_dir)
  99     cwd_path = os.path.realpath('.')
 100     while True:
 101         if os.path.realpath(path) == cwd_path:
 102             Print("Cannot use output directory '%s' since it is within the current directory '%s'" %
 103                   (path, cwd_path))
 104             sys.exit(1)
 105         parent = os.path.dirname(path)
 106         if parent == path:
 107             break
 108         path = parent
 109
 110 def ShowToolchainInfo(boards, toolchains, print_arch, print_prefix):
 111     """Show information about a the tool chain used by one or more boards
 112
 113     The function checks that all boards use the same toolchain.
 114
 115     Args:
 116         boards: Boards object containing selected boards
 117         toolchains: Toolchains object containing available toolchains
 118         print_arch: True to print ARCH value
 119         print_prefix: True to print CROSS_COMPILE value
 120
 121     Return:
 122         None on success, string error message otherwise
 123     """
 124     boards = boards.GetSelectedDict()
 125     tc_set = set()
 126     for brd in boards.values():
 127         tc_set.add(toolchains.Select(brd.arch))
 128     if len(tc_set) != 1:
 129         return 'Supplied boards must share one toolchain'
 130         return False
 131     tc = tc_set.pop()
 132     if print_arch:
 133         print(tc.GetEnvArgs(toolchain.VAR_ARCH))
 134     if print_prefix:
 135         print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 136     return None
 137
 138 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 139                clean_dir=False):
 140     """The main control code for buildman
 141
 142     Args:
 143         options: Command line options object
 144         args: Command line arguments (list of strings)
 145         toolchains: Toolchains to use - this should be a Toolchains()
 146                 object. If None, then it will be created and scanned
 147         make_func: Make function to use for the builder. This is called
 148                 to execute 'make'. If this is None, the normal function
 149                 will be used, which calls the 'make' tool with suitable
 150                 arguments. This setting is useful for tests.
 151         board: Boards() object to use, containing a list of available
 152                 boards. If this is None it will be created and scanned.
 153     """
 154     global builder
 155
 156     if options.full_help:
 157         pager = os.getenv('PAGER')
 158         if not pager:
 159             pager = 'more'
 160         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 161                              'README')
 162         command.Run(pager, fname)
 163         return 0
 164
 165     gitutil.Setup()
 166     col = terminal.Color()
 167
 168     options.git_dir = os.path.join(options.git, '.git')
 169
 170     no_toolchains = toolchains is None
 171     if no_toolchains:
 172         toolchains = toolchain.Toolchains(options.override_toolchain)
 173
 174     if options.fetch_arch:
 175         if options.fetch_arch == 'list':
 176             sorted_list = toolchains.ListArchs()
 177             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 178                             ' '.join(sorted_list)))
 179             return 0
 180         else:
 181             fetch_arch = options.fetch_arch
 182             if fetch_arch == 'all':
 183                 fetch_arch = ','.join(toolchains.ListArchs())
 184                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 185                                 fetch_arch))
 186             for arch in fetch_arch.split(','):
 187                 print()
 188                 ret = toolchains.FetchAndInstall(arch)
 189                 if ret:
 190                     return ret
 191             return 0
 192
 193     if no_toolchains:
 194         toolchains.GetSettings()
 195         toolchains.Scan(options.list_tool_chains and options.verbose)
 196     if options.list_tool_chains:
 197         toolchains.List()
 198         print()
 199         return 0
 200
 201     # Work out what subset of the boards we are building
 202     if not boards:
 203         if not os.path.exists(options.output_dir):
 204             os.makedirs(options.output_dir)
 205         board_file = os.path.join(options.output_dir, 'boards.cfg')
 206         genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 207         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 208         if status != 0:
 209             sys.exit("Failed to generate boards.cfg")
 210
 211         boards = board.Boards()
 212         boards.ReadBoards(board_file)
 213
 214     exclude = []
 215     if options.exclude:
 216         for arg in options.exclude:
 217             exclude += arg.split(',')
 218
 219     if options.boards:
 220         requested_boards = []
 221         for b in options.boards:
 222             requested_boards += b.split(',')
 223     else:
 224         requested_boards = None
 225     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 226                                                        requested_boards)
 227     selected = boards.GetSelected()
 228     if not len(selected):
 229         sys.exit(col.Color(col.RED, 'No matching boards found'))
 230
 231     if options.print_arch or options.print_prefix:
 232         err = ShowToolchainInfo(boards, toolchains, options.print_arch,
 233                                 options.print_prefix)
 234         if err:
 235             sys.exit(col.Color(col.RED, err))
 236         return 0
 237
 238     # Work out how many commits to build. We want to build everything on the
 239     # branch. We also build the upstream commit as a control so we can see
 240     # problems introduced by the first commit on the branch.
 241     count = options.count
 242     has_range = options.branch and '..' in options.branch
 243     if count == -1:
 244         if not options.branch:
 245             count = 1
 246         else:
 247             if has_range:
 248                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 249                                                          options.branch)
 250             else:
 251                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 252                                                           options.branch)
 253             if count is None:
 254                 sys.exit(col.Color(col.RED, msg))
 255             elif count == 0:
 256                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 257                                    options.branch))
 258             if msg:
 259                 print(col.Color(col.YELLOW, msg))
 260             count += 1   # Build upstream commit also
 261
 262     if not count:
 263         str = ("No commits found to process in branch '%s': "
 264                "set branch's upstream or use -c flag" % options.branch)
 265         sys.exit(col.Color(col.RED, str))
 266     if options.work_in_output:
 267         if len(selected) != 1:
 268             sys.exit(col.Color(col.RED,
 269                                '-w can only be used with a single board'))
 270         if count != 1:
 271             sys.exit(col.Color(col.RED,
 272                                '-w can only be used with a single commit'))
 273
 274     # Read the metadata from the commits. First look at the upstream commit,
 275     # then the ones in the branch. We would like to do something like
 276     # upstream/master~..branch but that isn't possible if upstream/master is
 277     # a merge commit (it will list all the commits that form part of the
 278     # merge)
 279     # Conflicting tags are not a problem for buildman, since it does not use
 280     # them. For example, Series-version is not useful for buildman. On the
 281     # other hand conflicting tags will cause an error. So allow later tags
 282     # to overwrite earlier ones by setting allow_overwrite=True
 283     if options.branch:
 284         if count == -1:
 285             if has_range:
 286                 range_expr = options.branch
 287             else:
 288                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 289                                                       options.branch)
 290             upstream_commit = gitutil.GetUpstream(options.git_dir,
 291                                                   options.branch)
 292             series = patchstream.GetMetaDataForList(upstream_commit,
 293                 options.git_dir, 1, series=None, allow_overwrite=True)
 294
 295             series = patchstream.GetMetaDataForList(range_expr,
 296                     options.git_dir, None, series, allow_overwrite=True)
 297         else:
 298             # Honour the count
 299             series = patchstream.GetMetaDataForList(options.branch,
 300                     options.git_dir, count, series=None, allow_overwrite=True)
 301     else:
 302         series = None
 303         if not options.dry_run:
 304             options.verbose = True
 305             if not options.summary:
 306                 options.show_errors = True
 307
 308     # By default we have one thread per CPU. But if there are not enough jobs
 309     # we can have fewer threads and use a high '-j' value for make.
 310     if not options.threads:
 311         options.threads = min(multiprocessing.cpu_count(), len(selected))
 312     if not options.jobs:
 313         options.jobs = max(1, (multiprocessing.cpu_count() +
 314                 len(selected) - 1) // len(selected))
 315
 316     if not options.step:
 317         options.step = len(series.commits) - 1
 318
 319     gnu_make = command.Output(os.path.join(options.git,
 320             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 321     if not gnu_make:
 322         sys.exit('GNU Make not found')
 323
 324     # Create a new builder with the selected options.
 325     output_dir = options.output_dir
 326     if options.branch:
 327         dirname = options.branch.replace('/', '_')
 328         # As a special case allow the board directory to be placed in the
 329         # output directory itself rather than any subdirectory.
 330         if not options.no_subdirs:
 331             output_dir = os.path.join(options.output_dir, dirname)
 332         if clean_dir and os.path.exists(output_dir):
 333             shutil.rmtree(output_dir)
 334     CheckOutputDir(output_dir)
 335     builder = Builder(toolchains, output_dir, options.git_dir,
 336             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 337             show_unknown=options.show_unknown, step=options.step,
 338             no_subdirs=options.no_subdirs, full_path=options.full_path,
 339             verbose_build=options.verbose_build,
 340             incremental=options.incremental,
 341             per_board_out_dir=options.per_board_out_dir,
 342             config_only=options.config_only,
 343             squash_config_y=not options.preserve_config_y,
 344             warnings_as_errors=options.warnings_as_errors,
 345             work_in_output=options.work_in_output)
 346     builder.force_config_on_failure = not options.quick
 347     if make_func:
 348         builder.do_make = make_func
 349
 350     # For a dry run, just show our actions as a sanity check
 351     if options.dry_run:
 352         ShowActions(series, why_selected, selected, builder, options,
 353                     board_warnings)
 354     else:
 355         builder.force_build = options.force_build
 356         builder.force_build_failures = options.force_build_failures
 357         builder.force_reconfig = options.force_reconfig
 358         builder.in_tree = options.in_tree
 359
 360         # Work out which boards to build
 361         board_selected = boards.GetSelectedDict()
 362
 363         if series:
 364             commits = series.commits
 365             # Number the commits for test purposes
 366             for commit in range(len(commits)):
 367                 commits[commit].sequence = commit
 368         else:
 369             commits = None
 370
 371         Print(GetActionSummary(options.summary, commits, board_selected,
 372                                 options))
 373
 374         # We can't show function sizes without board details at present
 375         if options.show_bloat:
 376             options.show_detail = True
 377         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 378                                   options.show_detail, options.show_bloat,
 379                                   options.list_error_boards,
 380                                   options.show_config,
 381                                   options.show_environment)
 382         if options.summary:
 383             builder.ShowSummary(commits, board_selected)
 384         else:
 385             fail, warned = builder.BuildBoards(commits, board_selected,
 386                                 options.keep_outputs, options.verbose)
 387             if fail:
 388                 return 128
 389             elif warned and not options.ignore_warnings:
 390                 return 129
 391     return 0