tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 from terminal import Print
  17 import toolchain
  18 import command
  19 import subprocess
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) // options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options,
  45                 board_warnings):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the list of boards
  52                 brought in by that argument. For example, 'arm' might bring
  53                 in 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59         board_warnings: List of warnings obtained from board selected
  60     """
  61     col = terminal.Color()
  62     print('Dry run, so not doing much. But I would do this:')
  63     print()
  64     if series:
  65         commits = series.commits
  66     else:
  67         commits = None
  68     print(GetActionSummary(False, commits, boards_selected,
  69             options))
  70     print('Build directory: %s' % builder.base_dir)
  71     if commits:
  72         for upto in range(0, len(series.commits), options.step):
  73             commit = series.commits[upto]
  74             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  75             print(commit.subject)
  76     print()
  77     for arg in why_selected:
  78         if arg != 'all':
  79             print(arg, ': %d boards' % len(why_selected[arg]))
  80             if options.verbose:
  81                 print('   %s' % ' '.join(why_selected[arg]))
  82     print(('Total boards to build for each commit: %d\n' %
  83             len(why_selected['all'])))
  84     if board_warnings:
  85         for warning in board_warnings:
  86             print(col.Color(col.YELLOW, warning))
  87
  88 def CheckOutputDir(output_dir):
  89     """Make sure that the output directory is not within the current directory
  90
  91     If we try to use an output directory which is within the current directory
  92     (which is assumed to hold the U-Boot source) we may end up deleting the
  93     U-Boot source code. Detect this and print an error in this case.
  94
  95     Args:
  96         output_dir: Output directory path to check
  97     """
  98     path = os.path.realpath(output_dir)
  99     cwd_path = os.path.realpath('.')
 100     while True:
 101         if os.path.realpath(path) == cwd_path:
 102             Print("Cannot use output directory '%s' since it is within the current directory '%s'" %
 103                   (path, cwd_path))
 104             sys.exit(1)
 105         parent = os.path.dirname(path)
 106         if parent == path:
 107             break
 108         path = parent
 109
 110 def ShowToolchainInfo(boards, toolchains, print_arch, print_prefix):
 111     """Show information about a the tool chain used by one or more boards
 112
 113     The function checks that all boards use the same toolchain.
 114
 115     Args:
 116         boards: Boards object containing selected boards
 117         toolchains: Toolchains object containing available toolchains
 118         print_arch: True to print ARCH value
 119         print_prefix: True to print CROSS_COMPILE value
 120
 121     Return:
 122         None on success, string error message otherwise
 123     """
 124     boards = boards.GetSelectedDict()
 125     tc_set = set()
 126     for brd in boards.values():
 127         tc_set.add(toolchains.Select(brd.arch))
 128     if len(tc_set) != 1:
 129         return 'Supplied boards must share one toolchain'
 130         return False
 131     tc = tc_set.pop()
 132     if print_arch:
 133         print(tc.GetEnvArgs(toolchain.VAR_ARCH))
 134     if print_prefix:
 135         print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 136     return None
 137
 138 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 139                clean_dir=False):
 140     """The main control code for buildman
 141
 142     Args:
 143         options: Command line options object
 144         args: Command line arguments (list of strings)
 145         toolchains: Toolchains to use - this should be a Toolchains()
 146                 object. If None, then it will be created and scanned
 147         make_func: Make function to use for the builder. This is called
 148                 to execute 'make'. If this is None, the normal function
 149                 will be used, which calls the 'make' tool with suitable
 150                 arguments. This setting is useful for tests.
 151         board: Boards() object to use, containing a list of available
 152                 boards. If this is None it will be created and scanned.
 153     """
 154     global builder
 155
 156     if options.full_help:
 157         pager = os.getenv('PAGER')
 158         if not pager:
 159             pager = 'more'
 160         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 161                              'README')
 162         command.Run(pager, fname)
 163         return 0
 164
 165     gitutil.Setup()
 166     col = terminal.Color()
 167
 168     options.git_dir = os.path.join(options.git, '.git')
 169
 170     no_toolchains = toolchains is None
 171     if no_toolchains:
 172         toolchains = toolchain.Toolchains(options.override_toolchain)
 173
 174     if options.fetch_arch:
 175         if options.fetch_arch == 'list':
 176             sorted_list = toolchains.ListArchs()
 177             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 178                             ' '.join(sorted_list)))
 179             return 0
 180         else:
 181             fetch_arch = options.fetch_arch
 182             if fetch_arch == 'all':
 183                 fetch_arch = ','.join(toolchains.ListArchs())
 184                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 185                                 fetch_arch))
 186             for arch in fetch_arch.split(','):
 187                 print()
 188                 ret = toolchains.FetchAndInstall(arch)
 189                 if ret:
 190                     return ret
 191             return 0
 192
 193     if no_toolchains:
 194         toolchains.GetSettings()
 195         toolchains.Scan(options.list_tool_chains and options.verbose)
 196     if options.list_tool_chains:
 197         toolchains.List()
 198         print()
 199         return 0
 200
 201     # Work out what subset of the boards we are building
 202     if not boards:
 203         if not os.path.exists(options.output_dir):
 204             os.makedirs(options.output_dir)
 205         board_file = os.path.join(options.output_dir, 'boards.cfg')
 206         genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 207         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 208         if status != 0:
 209             sys.exit("Failed to generate boards.cfg")
 210
 211         boards = board.Boards()
 212         boards.ReadBoards(board_file)
 213
 214     exclude = []
 215     if options.exclude:
 216         for arg in options.exclude:
 217             exclude += arg.split(',')
 218
 219     if options.boards:
 220         requested_boards = []
 221         for b in options.boards:
 222             requested_boards += b.split(',')
 223     else:
 224         requested_boards = None
 225     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 226                                                        requested_boards)
 227     selected = boards.GetSelected()
 228     if not len(selected):
 229         sys.exit(col.Color(col.RED, 'No matching boards found'))
 230
 231     if options.print_arch or options.print_prefix:
 232         err = ShowToolchainInfo(boards, toolchains, options.print_arch,
 233                                 options.print_prefix)
 234         if err:
 235             sys.exit(col.Color(col.RED, err))
 236         return 0
 237
 238     # Work out how many commits to build. We want to build everything on the
 239     # branch. We also build the upstream commit as a control so we can see
 240     # problems introduced by the first commit on the branch.
 241     count = options.count
 242     has_range = options.branch and '..' in options.branch
 243     if count == -1:
 244         if not options.branch:
 245             count = 1
 246         else:
 247             if has_range:
 248                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 249                                                          options.branch)
 250             else:
 251                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 252                                                           options.branch)
 253             if count is None:
 254                 sys.exit(col.Color(col.RED, msg))
 255             elif count == 0:
 256                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 257                                    options.branch))
 258             if msg:
 259                 print(col.Color(col.YELLOW, msg))
 260             count += 1   # Build upstream commit also
 261
 262     if not count:
 263         str = ("No commits found to process in branch '%s': "
 264                "set branch's upstream or use -c flag" % options.branch)
 265         sys.exit(col.Color(col.RED, str))
 266
 267     # Read the metadata from the commits. First look at the upstream commit,
 268     # then the ones in the branch. We would like to do something like
 269     # upstream/master~..branch but that isn't possible if upstream/master is
 270     # a merge commit (it will list all the commits that form part of the
 271     # merge)
 272     # Conflicting tags are not a problem for buildman, since it does not use
 273     # them. For example, Series-version is not useful for buildman. On the
 274     # other hand conflicting tags will cause an error. So allow later tags
 275     # to overwrite earlier ones by setting allow_overwrite=True
 276     if options.branch:
 277         if count == -1:
 278             if has_range:
 279                 range_expr = options.branch
 280             else:
 281                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 282                                                       options.branch)
 283             upstream_commit = gitutil.GetUpstream(options.git_dir,
 284                                                   options.branch)
 285             series = patchstream.GetMetaDataForList(upstream_commit,
 286                 options.git_dir, 1, series=None, allow_overwrite=True)
 287
 288             series = patchstream.GetMetaDataForList(range_expr,
 289                     options.git_dir, None, series, allow_overwrite=True)
 290         else:
 291             # Honour the count
 292             series = patchstream.GetMetaDataForList(options.branch,
 293                     options.git_dir, count, series=None, allow_overwrite=True)
 294     else:
 295         series = None
 296         if not options.dry_run:
 297             options.verbose = True
 298             if not options.summary:
 299                 options.show_errors = True
 300
 301     # By default we have one thread per CPU. But if there are not enough jobs
 302     # we can have fewer threads and use a high '-j' value for make.
 303     if not options.threads:
 304         options.threads = min(multiprocessing.cpu_count(), len(selected))
 305     if not options.jobs:
 306         options.jobs = max(1, (multiprocessing.cpu_count() +
 307                 len(selected) - 1) // len(selected))
 308
 309     if not options.step:
 310         options.step = len(series.commits) - 1
 311
 312     gnu_make = command.Output(os.path.join(options.git,
 313             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 314     if not gnu_make:
 315         sys.exit('GNU Make not found')
 316
 317     # Create a new builder with the selected options.
 318     output_dir = options.output_dir
 319     if options.branch:
 320         dirname = options.branch.replace('/', '_')
 321         # As a special case allow the board directory to be placed in the
 322         # output directory itself rather than any subdirectory.
 323         if not options.no_subdirs:
 324             output_dir = os.path.join(options.output_dir, dirname)
 325         if clean_dir and os.path.exists(output_dir):
 326             shutil.rmtree(output_dir)
 327     CheckOutputDir(output_dir)
 328     builder = Builder(toolchains, output_dir, options.git_dir,
 329             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 330             show_unknown=options.show_unknown, step=options.step,
 331             no_subdirs=options.no_subdirs, full_path=options.full_path,
 332             verbose_build=options.verbose_build,
 333             incremental=options.incremental,
 334             per_board_out_dir=options.per_board_out_dir,
 335             config_only=options.config_only,
 336             squash_config_y=not options.preserve_config_y,
 337             warnings_as_errors=options.warnings_as_errors)
 338     builder.force_config_on_failure = not options.quick
 339     if make_func:
 340         builder.do_make = make_func
 341
 342     # For a dry run, just show our actions as a sanity check
 343     if options.dry_run:
 344         ShowActions(series, why_selected, selected, builder, options,
 345                     board_warnings)
 346     else:
 347         builder.force_build = options.force_build
 348         builder.force_build_failures = options.force_build_failures
 349         builder.force_reconfig = options.force_reconfig
 350         builder.in_tree = options.in_tree
 351
 352         # Work out which boards to build
 353         board_selected = boards.GetSelectedDict()
 354
 355         if series:
 356             commits = series.commits
 357             # Number the commits for test purposes
 358             for commit in range(len(commits)):
 359                 commits[commit].sequence = commit
 360         else:
 361             commits = None
 362
 363         Print(GetActionSummary(options.summary, commits, board_selected,
 364                                 options))
 365
 366         # We can't show function sizes without board details at present
 367         if options.show_bloat:
 368             options.show_detail = True
 369         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 370                                   options.show_detail, options.show_bloat,
 371                                   options.list_error_boards,
 372                                   options.show_config,
 373                                   options.show_environment)
 374         if options.summary:
 375             builder.ShowSummary(commits, board_selected)
 376         else:
 377             fail, warned = builder.BuildBoards(commits, board_selected,
 378                                 options.keep_outputs, options.verbose)
 379             if fail:
 380                 return 128
 381             elif warned:
 382                 return 129
 383     return 0