tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 from terminal import Print
  17 import toolchain
  18 import command
  19 import subprocess
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) // options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options,
  45                 board_warnings):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the list of boards
  52                 brought in by that argument. For example, 'arm' might bring
  53                 in 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59         board_warnings: List of warnings obtained from board selected
  60     """
  61     col = terminal.Color()
  62     print('Dry run, so not doing much. But I would do this:')
  63     print()
  64     if series:
  65         commits = series.commits
  66     else:
  67         commits = None
  68     print(GetActionSummary(False, commits, boards_selected,
  69             options))
  70     print('Build directory: %s' % builder.base_dir)
  71     if commits:
  72         for upto in range(0, len(series.commits), options.step):
  73             commit = series.commits[upto]
  74             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  75             print(commit.subject)
  76     print()
  77     for arg in why_selected:
  78         if arg != 'all':
  79             print(arg, ': %d boards' % len(why_selected[arg]))
  80             if options.verbose:
  81                 print('   %s' % ' '.join(why_selected[arg]))
  82     print(('Total boards to build for each commit: %d\n' %
  83             len(why_selected['all'])))
  84     if board_warnings:
  85         for warning in board_warnings:
  86             print(col.Color(col.YELLOW, warning))
  87
  88 def CheckOutputDir(output_dir):
  89     """Make sure that the output directory is not within the current directory
  90
  91     If we try to use an output directory which is within the current directory
  92     (which is assumed to hold the U-Boot source) we may end up deleting the
  93     U-Boot source code. Detect this and print an error in this case.
  94
  95     Args:
  96         output_dir: Output directory path to check
  97     """
  98     path = os.path.realpath(output_dir)
  99     cwd_path = os.path.realpath('.')
 100     while True:
 101         if os.path.realpath(path) == cwd_path:
 102             Print("Cannot use output directory '%s' since it is within the current directory '%s'" %
 103                   (path, cwd_path))
 104             sys.exit(1)
 105         parent = os.path.dirname(path)
 106         if parent == path:
 107             break
 108         path = parent
 109
 110 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 111                clean_dir=False):
 112     """The main control code for buildman
 113
 114     Args:
 115         options: Command line options object
 116         args: Command line arguments (list of strings)
 117         toolchains: Toolchains to use - this should be a Toolchains()
 118                 object. If None, then it will be created and scanned
 119         make_func: Make function to use for the builder. This is called
 120                 to execute 'make'. If this is None, the normal function
 121                 will be used, which calls the 'make' tool with suitable
 122                 arguments. This setting is useful for tests.
 123         board: Boards() object to use, containing a list of available
 124                 boards. If this is None it will be created and scanned.
 125     """
 126     global builder
 127
 128     if options.full_help:
 129         pager = os.getenv('PAGER')
 130         if not pager:
 131             pager = 'more'
 132         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 133                              'README')
 134         command.Run(pager, fname)
 135         return 0
 136
 137     gitutil.Setup()
 138     col = terminal.Color()
 139
 140     options.git_dir = os.path.join(options.git, '.git')
 141
 142     no_toolchains = toolchains is None
 143     if no_toolchains:
 144         toolchains = toolchain.Toolchains(options.override_toolchain)
 145
 146     if options.fetch_arch:
 147         if options.fetch_arch == 'list':
 148             sorted_list = toolchains.ListArchs()
 149             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 150                             ' '.join(sorted_list)))
 151             return 0
 152         else:
 153             fetch_arch = options.fetch_arch
 154             if fetch_arch == 'all':
 155                 fetch_arch = ','.join(toolchains.ListArchs())
 156                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 157                                 fetch_arch))
 158             for arch in fetch_arch.split(','):
 159                 print()
 160                 ret = toolchains.FetchAndInstall(arch)
 161                 if ret:
 162                     return ret
 163             return 0
 164
 165     if no_toolchains:
 166         toolchains.GetSettings()
 167         toolchains.Scan(options.list_tool_chains and options.verbose)
 168     if options.list_tool_chains:
 169         toolchains.List()
 170         print()
 171         return 0
 172
 173     # Work out what subset of the boards we are building
 174     if not boards:
 175         if not os.path.exists(options.output_dir):
 176             os.makedirs(options.output_dir)
 177         board_file = os.path.join(options.output_dir, 'boards.cfg')
 178         genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 179         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 180         if status != 0:
 181             sys.exit("Failed to generate boards.cfg")
 182
 183         boards = board.Boards()
 184         boards.ReadBoards(board_file)
 185
 186     exclude = []
 187     if options.exclude:
 188         for arg in options.exclude:
 189             exclude += arg.split(',')
 190
 191     if options.boards:
 192         requested_boards = []
 193         for b in options.boards:
 194             requested_boards += b.split(',')
 195     else:
 196         requested_boards = None
 197     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 198                                                        requested_boards)
 199     selected = boards.GetSelected()
 200     if not len(selected):
 201         sys.exit(col.Color(col.RED, 'No matching boards found'))
 202
 203     # Work out how many commits to build. We want to build everything on the
 204     # branch. We also build the upstream commit as a control so we can see
 205     # problems introduced by the first commit on the branch.
 206     count = options.count
 207     has_range = options.branch and '..' in options.branch
 208     if count == -1:
 209         if not options.branch:
 210             count = 1
 211         else:
 212             if has_range:
 213                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 214                                                          options.branch)
 215             else:
 216                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 217                                                           options.branch)
 218             if count is None:
 219                 sys.exit(col.Color(col.RED, msg))
 220             elif count == 0:
 221                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 222                                    options.branch))
 223             if msg:
 224                 print(col.Color(col.YELLOW, msg))
 225             count += 1   # Build upstream commit also
 226
 227     if not count:
 228         str = ("No commits found to process in branch '%s': "
 229                "set branch's upstream or use -c flag" % options.branch)
 230         sys.exit(col.Color(col.RED, str))
 231
 232     # Read the metadata from the commits. First look at the upstream commit,
 233     # then the ones in the branch. We would like to do something like
 234     # upstream/master~..branch but that isn't possible if upstream/master is
 235     # a merge commit (it will list all the commits that form part of the
 236     # merge)
 237     # Conflicting tags are not a problem for buildman, since it does not use
 238     # them. For example, Series-version is not useful for buildman. On the
 239     # other hand conflicting tags will cause an error. So allow later tags
 240     # to overwrite earlier ones by setting allow_overwrite=True
 241     if options.branch:
 242         if count == -1:
 243             if has_range:
 244                 range_expr = options.branch
 245             else:
 246                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 247                                                       options.branch)
 248             upstream_commit = gitutil.GetUpstream(options.git_dir,
 249                                                   options.branch)
 250             series = patchstream.GetMetaDataForList(upstream_commit,
 251                 options.git_dir, 1, series=None, allow_overwrite=True)
 252
 253             series = patchstream.GetMetaDataForList(range_expr,
 254                     options.git_dir, None, series, allow_overwrite=True)
 255         else:
 256             # Honour the count
 257             series = patchstream.GetMetaDataForList(options.branch,
 258                     options.git_dir, count, series=None, allow_overwrite=True)
 259     else:
 260         series = None
 261         if not options.dry_run:
 262             options.verbose = True
 263             if not options.summary:
 264                 options.show_errors = True
 265
 266     # By default we have one thread per CPU. But if there are not enough jobs
 267     # we can have fewer threads and use a high '-j' value for make.
 268     if not options.threads:
 269         options.threads = min(multiprocessing.cpu_count(), len(selected))
 270     if not options.jobs:
 271         options.jobs = max(1, (multiprocessing.cpu_count() +
 272                 len(selected) - 1) // len(selected))
 273
 274     if not options.step:
 275         options.step = len(series.commits) - 1
 276
 277     gnu_make = command.Output(os.path.join(options.git,
 278             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 279     if not gnu_make:
 280         sys.exit('GNU Make not found')
 281
 282     # Create a new builder with the selected options.
 283     output_dir = options.output_dir
 284     if options.branch:
 285         dirname = options.branch.replace('/', '_')
 286         # As a special case allow the board directory to be placed in the
 287         # output directory itself rather than any subdirectory.
 288         if not options.no_subdirs:
 289             output_dir = os.path.join(options.output_dir, dirname)
 290         if clean_dir and os.path.exists(output_dir):
 291             shutil.rmtree(output_dir)
 292     CheckOutputDir(output_dir)
 293     builder = Builder(toolchains, output_dir, options.git_dir,
 294             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 295             show_unknown=options.show_unknown, step=options.step,
 296             no_subdirs=options.no_subdirs, full_path=options.full_path,
 297             verbose_build=options.verbose_build,
 298             incremental=options.incremental,
 299             per_board_out_dir=options.per_board_out_dir,
 300             config_only=options.config_only,
 301             squash_config_y=not options.preserve_config_y,
 302             warnings_as_errors=options.warnings_as_errors)
 303     builder.force_config_on_failure = not options.quick
 304     if make_func:
 305         builder.do_make = make_func
 306
 307     # For a dry run, just show our actions as a sanity check
 308     if options.dry_run:
 309         ShowActions(series, why_selected, selected, builder, options,
 310                     board_warnings)
 311     else:
 312         builder.force_build = options.force_build
 313         builder.force_build_failures = options.force_build_failures
 314         builder.force_reconfig = options.force_reconfig
 315         builder.in_tree = options.in_tree
 316
 317         # Work out which boards to build
 318         board_selected = boards.GetSelectedDict()
 319
 320         if series:
 321             commits = series.commits
 322             # Number the commits for test purposes
 323             for commit in range(len(commits)):
 324                 commits[commit].sequence = commit
 325         else:
 326             commits = None
 327
 328         Print(GetActionSummary(options.summary, commits, board_selected,
 329                                 options))
 330
 331         # We can't show function sizes without board details at present
 332         if options.show_bloat:
 333             options.show_detail = True
 334         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 335                                   options.show_detail, options.show_bloat,
 336                                   options.list_error_boards,
 337                                   options.show_config,
 338                                   options.show_environment)
 339         if options.summary:
 340             builder.ShowSummary(commits, board_selected)
 341         else:
 342             fail, warned = builder.BuildBoards(commits, board_selected,
 343                                 options.keep_outputs, options.verbose)
 344             if fail:
 345                 return 128
 346             elif warned:
 347                 return 129
 348     return 0