tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 from terminal import Print
  17 import toolchain
  18 import command
  19 import subprocess
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) // options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options,
  45                 board_warnings):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the list of boards
  52                 brought in by that argument. For example, 'arm' might bring
  53                 in 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59         board_warnings: List of warnings obtained from board selected
  60     """
  61     col = terminal.Color()
  62     print('Dry run, so not doing much. But I would do this:')
  63     print()
  64     if series:
  65         commits = series.commits
  66     else:
  67         commits = None
  68     print(GetActionSummary(False, commits, boards_selected,
  69             options))
  70     print('Build directory: %s' % builder.base_dir)
  71     if commits:
  72         for upto in range(0, len(series.commits), options.step):
  73             commit = series.commits[upto]
  74             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  75             print(commit.subject)
  76     print()
  77     for arg in why_selected:
  78         if arg != 'all':
  79             print(arg, ': %d boards' % len(why_selected[arg]))
  80             if options.verbose:
  81                 print('   %s' % ' '.join(why_selected[arg]))
  82     print(('Total boards to build for each commit: %d\n' %
  83             len(why_selected['all'])))
  84     if board_warnings:
  85         for warning in board_warnings:
  86             print(col.Color(col.YELLOW, warning))
  87
  88 def CheckOutputDir(output_dir):
  89     """Make sure that the output directory is not within the current directory
  90
  91     If we try to use an output directory which is within the current directory
  92     (which is assumed to hold the U-Boot source) we may end up deleting the
  93     U-Boot source code. Detect this and print an error in this case.
  94
  95     Args:
  96         output_dir: Output directory path to check
  97     """
  98     path = os.path.realpath(output_dir)
  99     cwd_path = os.path.realpath('.')
 100     while True:
 101         if os.path.realpath(path) == cwd_path:
 102             Print("Cannot use output directory '%s' since it is within the current directory '%s'" %
 103                   (path, cwd_path))
 104             sys.exit(1)
 105         parent = os.path.dirname(path)
 106         if parent == path:
 107             break
 108         path = parent
 109
 110 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 111                clean_dir=False):
 112     """The main control code for buildman
 113
 114     Args:
 115         options: Command line options object
 116         args: Command line arguments (list of strings)
 117         toolchains: Toolchains to use - this should be a Toolchains()
 118                 object. If None, then it will be created and scanned
 119         make_func: Make function to use for the builder. This is called
 120                 to execute 'make'. If this is None, the normal function
 121                 will be used, which calls the 'make' tool with suitable
 122                 arguments. This setting is useful for tests.
 123         board: Boards() object to use, containing a list of available
 124                 boards. If this is None it will be created and scanned.
 125     """
 126     global builder
 127
 128     if options.full_help:
 129         pager = os.getenv('PAGER')
 130         if not pager:
 131             pager = 'more'
 132         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 133                              'README')
 134         command.Run(pager, fname)
 135         return 0
 136
 137     gitutil.Setup()
 138     col = terminal.Color()
 139
 140     options.git_dir = os.path.join(options.git, '.git')
 141
 142     no_toolchains = toolchains is None
 143     if no_toolchains:
 144         toolchains = toolchain.Toolchains(options.override_toolchain)
 145
 146     if options.fetch_arch:
 147         if options.fetch_arch == 'list':
 148             sorted_list = toolchains.ListArchs()
 149             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 150                             ' '.join(sorted_list)))
 151             return 0
 152         else:
 153             fetch_arch = options.fetch_arch
 154             if fetch_arch == 'all':
 155                 fetch_arch = ','.join(toolchains.ListArchs())
 156                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 157                                 fetch_arch))
 158             for arch in fetch_arch.split(','):
 159                 print()
 160                 ret = toolchains.FetchAndInstall(arch)
 161                 if ret:
 162                     return ret
 163             return 0
 164
 165     if no_toolchains:
 166         toolchains.GetSettings()
 167         toolchains.Scan(options.list_tool_chains and options.verbose)
 168     if options.list_tool_chains:
 169         toolchains.List()
 170         print()
 171         return 0
 172
 173     # Work out how many commits to build. We want to build everything on the
 174     # branch. We also build the upstream commit as a control so we can see
 175     # problems introduced by the first commit on the branch.
 176     count = options.count
 177     has_range = options.branch and '..' in options.branch
 178     if count == -1:
 179         if not options.branch:
 180             count = 1
 181         else:
 182             if has_range:
 183                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 184                                                          options.branch)
 185             else:
 186                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 187                                                           options.branch)
 188             if count is None:
 189                 sys.exit(col.Color(col.RED, msg))
 190             elif count == 0:
 191                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 192                                    options.branch))
 193             if msg:
 194                 print(col.Color(col.YELLOW, msg))
 195             count += 1   # Build upstream commit also
 196
 197     if not count:
 198         str = ("No commits found to process in branch '%s': "
 199                "set branch's upstream or use -c flag" % options.branch)
 200         sys.exit(col.Color(col.RED, str))
 201
 202     # Work out what subset of the boards we are building
 203     if not boards:
 204         if not os.path.exists(options.output_dir):
 205             os.makedirs(options.output_dir)
 206         board_file = os.path.join(options.output_dir, 'boards.cfg')
 207         genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 208         status = subprocess.call([genboardscfg, '-o', board_file])
 209         if status != 0:
 210             sys.exit("Failed to generate boards.cfg")
 211
 212         boards = board.Boards()
 213         boards.ReadBoards(board_file)
 214
 215     exclude = []
 216     if options.exclude:
 217         for arg in options.exclude:
 218             exclude += arg.split(',')
 219
 220
 221     if options.boards:
 222         requested_boards = []
 223         for b in options.boards:
 224             requested_boards += b.split(',')
 225     else:
 226         requested_boards = None
 227     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 228                                                        requested_boards)
 229     selected = boards.GetSelected()
 230     if not len(selected):
 231         sys.exit(col.Color(col.RED, 'No matching boards found'))
 232
 233     # Read the metadata from the commits. First look at the upstream commit,
 234     # then the ones in the branch. We would like to do something like
 235     # upstream/master~..branch but that isn't possible if upstream/master is
 236     # a merge commit (it will list all the commits that form part of the
 237     # merge)
 238     # Conflicting tags are not a problem for buildman, since it does not use
 239     # them. For example, Series-version is not useful for buildman. On the
 240     # other hand conflicting tags will cause an error. So allow later tags
 241     # to overwrite earlier ones by setting allow_overwrite=True
 242     if options.branch:
 243         if count == -1:
 244             if has_range:
 245                 range_expr = options.branch
 246             else:
 247                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 248                                                       options.branch)
 249             upstream_commit = gitutil.GetUpstream(options.git_dir,
 250                                                   options.branch)
 251             series = patchstream.GetMetaDataForList(upstream_commit,
 252                 options.git_dir, 1, series=None, allow_overwrite=True)
 253
 254             series = patchstream.GetMetaDataForList(range_expr,
 255                     options.git_dir, None, series, allow_overwrite=True)
 256         else:
 257             # Honour the count
 258             series = patchstream.GetMetaDataForList(options.branch,
 259                     options.git_dir, count, series=None, allow_overwrite=True)
 260     else:
 261         series = None
 262         if not options.dry_run:
 263             options.verbose = True
 264             if not options.summary:
 265                 options.show_errors = True
 266
 267     # By default we have one thread per CPU. But if there are not enough jobs
 268     # we can have fewer threads and use a high '-j' value for make.
 269     if not options.threads:
 270         options.threads = min(multiprocessing.cpu_count(), len(selected))
 271     if not options.jobs:
 272         options.jobs = max(1, (multiprocessing.cpu_count() +
 273                 len(selected) - 1) // len(selected))
 274
 275     if not options.step:
 276         options.step = len(series.commits) - 1
 277
 278     gnu_make = command.Output(os.path.join(options.git,
 279             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 280     if not gnu_make:
 281         sys.exit('GNU Make not found')
 282
 283     # Create a new builder with the selected options.
 284     output_dir = options.output_dir
 285     if options.branch:
 286         dirname = options.branch.replace('/', '_')
 287         # As a special case allow the board directory to be placed in the
 288         # output directory itself rather than any subdirectory.
 289         if not options.no_subdirs:
 290             output_dir = os.path.join(options.output_dir, dirname)
 291         if clean_dir and os.path.exists(output_dir):
 292             shutil.rmtree(output_dir)
 293     CheckOutputDir(output_dir)
 294     builder = Builder(toolchains, output_dir, options.git_dir,
 295             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 296             show_unknown=options.show_unknown, step=options.step,
 297             no_subdirs=options.no_subdirs, full_path=options.full_path,
 298             verbose_build=options.verbose_build,
 299             incremental=options.incremental,
 300             per_board_out_dir=options.per_board_out_dir,
 301             config_only=options.config_only,
 302             squash_config_y=not options.preserve_config_y,
 303             warnings_as_errors=options.warnings_as_errors)
 304     builder.force_config_on_failure = not options.quick
 305     if make_func:
 306         builder.do_make = make_func
 307
 308     # For a dry run, just show our actions as a sanity check
 309     if options.dry_run:
 310         ShowActions(series, why_selected, selected, builder, options,
 311                     board_warnings)
 312     else:
 313         builder.force_build = options.force_build
 314         builder.force_build_failures = options.force_build_failures
 315         builder.force_reconfig = options.force_reconfig
 316         builder.in_tree = options.in_tree
 317
 318         # Work out which boards to build
 319         board_selected = boards.GetSelectedDict()
 320
 321         if series:
 322             commits = series.commits
 323             # Number the commits for test purposes
 324             for commit in range(len(commits)):
 325                 commits[commit].sequence = commit
 326         else:
 327             commits = None
 328
 329         Print(GetActionSummary(options.summary, commits, board_selected,
 330                                 options))
 331
 332         # We can't show function sizes without board details at present
 333         if options.show_bloat:
 334             options.show_detail = True
 335         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 336                                   options.show_detail, options.show_bloat,
 337                                   options.list_error_boards,
 338                                   options.show_config,
 339                                   options.show_environment)
 340         if options.summary:
 341             builder.ShowSummary(commits, board_selected)
 342         else:
 343             fail, warned = builder.BuildBoards(commits, board_selected,
 344                                 options.keep_outputs, options.verbose)
 345             if fail:
 346                 return 128
 347             elif warned:
 348                 return 129
 349     return 0