tools/buildman/builderthread.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4
   5 import errno
   6 import glob
   7 import os
   8 import shutil
   9 import sys
  10 import threading
  11
  12 import command
  13 import gitutil
  14
  15 RETURN_CODE_RETRY = -1
  16
  17 def Mkdir(dirname, parents = False):
  18     """Make a directory if it doesn't already exist.
  19
  20     Args:
  21         dirname: Directory to create
  22     """
  23     try:
  24         if parents:
  25             os.makedirs(dirname)
  26         else:
  27             os.mkdir(dirname)
  28     except OSError as err:
  29         if err.errno == errno.EEXIST:
  30             if os.path.realpath('.') == os.path.realpath(dirname):
  31                 print("Cannot create the current working directory '%s'!" % dirname)
  32                 sys.exit(1)
  33             pass
  34         else:
  35             raise
  36
  37 class BuilderJob:
  38     """Holds information about a job to be performed by a thread
  39
  40     Members:
  41         board: Board object to build
  42         commits: List of Commit objects to build
  43         keep_outputs: True to save build output files
  44         step: 1 to process every commit, n to process every nth commit
  45     """
  46     def __init__(self):
  47         self.board = None
  48         self.commits = []
  49         self.keep_outputs = False
  50         self.step = 1
  51
  52
  53 class ResultThread(threading.Thread):
  54     """This thread processes results from builder threads.
  55
  56     It simply passes the results on to the builder. There is only one
  57     result thread, and this helps to serialise the build output.
  58     """
  59     def __init__(self, builder):
  60         """Set up a new result thread
  61
  62         Args:
  63             builder: Builder which will be sent each result
  64         """
  65         threading.Thread.__init__(self)
  66         self.builder = builder
  67
  68     def run(self):
  69         """Called to start up the result thread.
  70
  71         We collect the next result job and pass it on to the build.
  72         """
  73         while True:
  74             result = self.builder.out_queue.get()
  75             self.builder.ProcessResult(result)
  76             self.builder.out_queue.task_done()
  77
  78
  79 class BuilderThread(threading.Thread):
  80     """This thread builds U-Boot for a particular board.
  81
  82     An input queue provides each new job. We run 'make' to build U-Boot
  83     and then pass the results on to the output queue.
  84
  85     Members:
  86         builder: The builder which contains information we might need
  87         thread_num: Our thread number (0-n-1), used to decide on a
  88                 temporary directory
  89     """
  90     def __init__(self, builder, thread_num, incremental, per_board_out_dir):
  91         """Set up a new builder thread"""
  92         threading.Thread.__init__(self)
  93         self.builder = builder
  94         self.thread_num = thread_num
  95         self.incremental = incremental
  96         self.per_board_out_dir = per_board_out_dir
  97
  98     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
  99         """Run 'make' on a particular commit and board.
 100
 101         The source code will already be checked out, so the 'commit'
 102         argument is only for information.
 103
 104         Args:
 105             commit: Commit object that is being built
 106             brd: Board object that is being built
 107             stage: Stage of the build. Valid stages are:
 108                         mrproper - can be called to clean source
 109                         config - called to configure for a board
 110                         build - the main make invocation - it does the build
 111             args: A list of arguments to pass to 'make'
 112             kwargs: A list of keyword arguments to pass to command.RunPipe()
 113
 114         Returns:
 115             CommandResult object
 116         """
 117         return self.builder.do_make(commit, brd, stage, cwd, *args,
 118                 **kwargs)
 119
 120     def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
 121                   force_build, force_build_failures):
 122         """Build a particular commit.
 123
 124         If the build is already done, and we are not forcing a build, we skip
 125         the build and just return the previously-saved results.
 126
 127         Args:
 128             commit_upto: Commit number to build (0...n-1)
 129             brd: Board object to build
 130             work_dir: Directory to which the source will be checked out
 131             do_config: True to run a make <board>_defconfig on the source
 132             config_only: Only configure the source, do not build it
 133             force_build: Force a build even if one was previously done
 134             force_build_failures: Force a bulid if the previous result showed
 135                 failure
 136
 137         Returns:
 138             tuple containing:
 139                 - CommandResult object containing the results of the build
 140                 - boolean indicating whether 'make config' is still needed
 141         """
 142         # Create a default result - it will be overwritte by the call to
 143         # self.Make() below, in the event that we do a build.
 144         result = command.CommandResult()
 145         result.return_code = 0
 146         if self.builder.in_tree:
 147             out_dir = work_dir
 148         else:
 149             if self.per_board_out_dir:
 150                 out_rel_dir = os.path.join('..', brd.target)
 151             else:
 152                 out_rel_dir = 'build'
 153             out_dir = os.path.join(work_dir, out_rel_dir)
 154
 155         # Check if the job was already completed last time
 156         done_file = self.builder.GetDoneFile(commit_upto, brd.target)
 157         result.already_done = os.path.exists(done_file)
 158         will_build = (force_build or force_build_failures or
 159             not result.already_done)
 160         if result.already_done:
 161             # Get the return code from that build and use it
 162             with open(done_file, 'r') as fd:
 163                 try:
 164                     result.return_code = int(fd.readline())
 165                 except ValueError:
 166                     # The file may be empty due to running out of disk space.
 167                     # Try a rebuild
 168                     result.return_code = RETURN_CODE_RETRY
 169
 170             # Check the signal that the build needs to be retried
 171             if result.return_code == RETURN_CODE_RETRY:
 172                 will_build = True
 173             elif will_build:
 174                 err_file = self.builder.GetErrFile(commit_upto, brd.target)
 175                 if os.path.exists(err_file) and os.stat(err_file).st_size:
 176                     result.stderr = 'bad'
 177                 elif not force_build:
 178                     # The build passed, so no need to build it again
 179                     will_build = False
 180
 181         if will_build:
 182             # We are going to have to build it. First, get a toolchain
 183             if not self.toolchain:
 184                 try:
 185                     self.toolchain = self.builder.toolchains.Select(brd.arch)
 186                 except ValueError as err:
 187                     result.return_code = 10
 188                     result.stdout = ''
 189                     result.stderr = str(err)
 190                     # TODO(sjg@chromium.org): This gets swallowed, but needs
 191                     # to be reported.
 192
 193             if self.toolchain:
 194                 # Checkout the right commit
 195                 if self.builder.commits:
 196                     commit = self.builder.commits[commit_upto]
 197                     if self.builder.checkout:
 198                         git_dir = os.path.join(work_dir, '.git')
 199                         gitutil.Checkout(commit.hash, git_dir, work_dir,
 200                                          force=True)
 201                 else:
 202                     commit = 'current'
 203
 204                 # Set up the environment and command line
 205                 env = self.toolchain.MakeEnvironment(self.builder.full_path)
 206                 Mkdir(out_dir)
 207                 args = []
 208                 cwd = work_dir
 209                 src_dir = os.path.realpath(work_dir)
 210                 if not self.builder.in_tree:
 211                     if commit_upto is None:
 212                         # In this case we are building in the original source
 213                         # directory (i.e. the current directory where buildman
 214                         # is invoked. The output directory is set to this
 215                         # thread's selected work directory.
 216                         #
 217                         # Symlinks can confuse U-Boot's Makefile since
 218                         # we may use '..' in our path, so remove them.
 219                         out_dir = os.path.realpath(out_dir)
 220                         args.append('O=%s' % out_dir)
 221                         cwd = None
 222                         src_dir = os.getcwd()
 223                     else:
 224                         args.append('O=%s' % out_rel_dir)
 225                 if self.builder.verbose_build:
 226                     args.append('V=1')
 227                 else:
 228                     args.append('-s')
 229                 if self.builder.num_jobs is not None:
 230                     args.extend(['-j', str(self.builder.num_jobs)])
 231                 if self.builder.warnings_as_errors:
 232                     args.append('KCFLAGS=-Werror')
 233                 config_args = ['%s_defconfig' % brd.target]
 234                 config_out = ''
 235                 args.extend(self.builder.toolchains.GetMakeArguments(brd))
 236                 args.extend(self.toolchain.MakeArgs())
 237
 238                 # If we need to reconfigure, do that now
 239                 if do_config:
 240                     config_out = ''
 241                     if not self.incremental:
 242                         result = self.Make(commit, brd, 'mrproper', cwd,
 243                                 'mrproper', *args, env=env)
 244                         config_out += result.combined
 245                     result = self.Make(commit, brd, 'config', cwd,
 246                             *(args + config_args), env=env)
 247                     config_out += result.combined
 248                     do_config = False   # No need to configure next time
 249                 if result.return_code == 0:
 250                     if config_only:
 251                         args.append('cfg')
 252                     result = self.Make(commit, brd, 'build', cwd, *args,
 253                             env=env)
 254                 result.stderr = result.stderr.replace(src_dir + '/', '')
 255                 if self.builder.verbose_build:
 256                     result.stdout = config_out + result.stdout
 257             else:
 258                 result.return_code = 1
 259                 result.stderr = 'No tool chain for %s\n' % brd.arch
 260             result.already_done = False
 261
 262         result.toolchain = self.toolchain
 263         result.brd = brd
 264         result.commit_upto = commit_upto
 265         result.out_dir = out_dir
 266         return result, do_config
 267
 268     def _WriteResult(self, result, keep_outputs):
 269         """Write a built result to the output directory.
 270
 271         Args:
 272             result: CommandResult object containing result to write
 273             keep_outputs: True to store the output binaries, False
 274                 to delete them
 275         """
 276         # Fatal error
 277         if result.return_code < 0:
 278             return
 279
 280         # If we think this might have been aborted with Ctrl-C, record the
 281         # failure but not that we are 'done' with this board. A retry may fix
 282         # it.
 283         maybe_aborted =  result.stderr and 'No child processes' in result.stderr
 284
 285         if result.already_done:
 286             return
 287
 288         # Write the output and stderr
 289         output_dir = self.builder._GetOutputDir(result.commit_upto)
 290         Mkdir(output_dir)
 291         build_dir = self.builder.GetBuildDir(result.commit_upto,
 292                 result.brd.target)
 293         Mkdir(build_dir)
 294
 295         outfile = os.path.join(build_dir, 'log')
 296         with open(outfile, 'w') as fd:
 297             if result.stdout:
 298                 fd.write(result.stdout)
 299
 300         errfile = self.builder.GetErrFile(result.commit_upto,
 301                 result.brd.target)
 302         if result.stderr:
 303             with open(errfile, 'w') as fd:
 304                 fd.write(result.stderr)
 305         elif os.path.exists(errfile):
 306             os.remove(errfile)
 307
 308         if result.toolchain:
 309             # Write the build result and toolchain information.
 310             done_file = self.builder.GetDoneFile(result.commit_upto,
 311                     result.brd.target)
 312             with open(done_file, 'w') as fd:
 313                 if maybe_aborted:
 314                     # Special code to indicate we need to retry
 315                     fd.write('%s' % RETURN_CODE_RETRY)
 316                 else:
 317                     fd.write('%s' % result.return_code)
 318             with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
 319                 print('gcc', result.toolchain.gcc, file=fd)
 320                 print('path', result.toolchain.path, file=fd)
 321                 print('cross', result.toolchain.cross, file=fd)
 322                 print('arch', result.toolchain.arch, file=fd)
 323                 fd.write('%s' % result.return_code)
 324
 325             # Write out the image and function size information and an objdump
 326             env = result.toolchain.MakeEnvironment(self.builder.full_path)
 327             with open(os.path.join(build_dir, 'env'), 'w') as fd:
 328                 for var in sorted(env.keys()):
 329                     print('%s="%s"' % (var, env[var]), file=fd)
 330             lines = []
 331             for fname in ['u-boot', 'spl/u-boot-spl']:
 332                 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
 333                 nm_result = command.RunPipe([cmd], capture=True,
 334                         capture_stderr=True, cwd=result.out_dir,
 335                         raise_on_error=False, env=env)
 336                 if nm_result.stdout:
 337                     nm = self.builder.GetFuncSizesFile(result.commit_upto,
 338                                     result.brd.target, fname)
 339                     with open(nm, 'w') as fd:
 340                         print(nm_result.stdout, end=' ', file=fd)
 341
 342                 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
 343                 dump_result = command.RunPipe([cmd], capture=True,
 344                         capture_stderr=True, cwd=result.out_dir,
 345                         raise_on_error=False, env=env)
 346                 rodata_size = ''
 347                 if dump_result.stdout:
 348                     objdump = self.builder.GetObjdumpFile(result.commit_upto,
 349                                     result.brd.target, fname)
 350                     with open(objdump, 'w') as fd:
 351                         print(dump_result.stdout, end=' ', file=fd)
 352                     for line in dump_result.stdout.splitlines():
 353                         fields = line.split()
 354                         if len(fields) > 5 and fields[1] == '.rodata':
 355                             rodata_size = fields[2]
 356
 357                 cmd = ['%ssize' % self.toolchain.cross, fname]
 358                 size_result = command.RunPipe([cmd], capture=True,
 359                         capture_stderr=True, cwd=result.out_dir,
 360                         raise_on_error=False, env=env)
 361                 if size_result.stdout:
 362                     lines.append(size_result.stdout.splitlines()[1] + ' ' +
 363                                  rodata_size)
 364
 365             # Extract the environment from U-Boot and dump it out
 366             cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary',
 367                    '-j', '.rodata.default_environment',
 368                    'env/built-in.o', 'uboot.env']
 369             command.RunPipe([cmd], capture=True,
 370                             capture_stderr=True, cwd=result.out_dir,
 371                             raise_on_error=False, env=env)
 372             ubootenv = os.path.join(result.out_dir, 'uboot.env')
 373             self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
 374
 375             # Write out the image sizes file. This is similar to the output
 376             # of binutil's 'size' utility, but it omits the header line and
 377             # adds an additional hex value at the end of each line for the
 378             # rodata size
 379             if len(lines):
 380                 sizes = self.builder.GetSizesFile(result.commit_upto,
 381                                 result.brd.target)
 382                 with open(sizes, 'w') as fd:
 383                     print('\n'.join(lines), file=fd)
 384
 385         # Write out the configuration files, with a special case for SPL
 386         for dirname in ['', 'spl', 'tpl']:
 387             self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
 388                 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
 389                 'include/autoconf.mk', 'include/generated/autoconf.h'])
 390
 391         # Now write the actual build output
 392         if keep_outputs:
 393             self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
 394                 '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
 395                 'spl/u-boot-spl*'])
 396
 397     def CopyFiles(self, out_dir, build_dir, dirname, patterns):
 398         """Copy files from the build directory to the output.
 399
 400         Args:
 401             out_dir: Path to output directory containing the files
 402             build_dir: Place to copy the files
 403             dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
 404             patterns: A list of filenames (strings) to copy, each relative
 405                to the build directory
 406         """
 407         for pattern in patterns:
 408             file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
 409             for fname in file_list:
 410                 target = os.path.basename(fname)
 411                 if dirname:
 412                     base, ext = os.path.splitext(target)
 413                     if ext:
 414                         target = '%s-%s%s' % (base, dirname, ext)
 415                 shutil.copy(fname, os.path.join(build_dir, target))
 416
 417     def RunJob(self, job):
 418         """Run a single job
 419
 420         A job consists of a building a list of commits for a particular board.
 421
 422         Args:
 423             job: Job to build
 424         """
 425         brd = job.board
 426         work_dir = self.builder.GetThreadDir(self.thread_num)
 427         self.toolchain = None
 428         if job.commits:
 429             # Run 'make board_defconfig' on the first commit
 430             do_config = True
 431             commit_upto  = 0
 432             force_build = False
 433             for commit_upto in range(0, len(job.commits), job.step):
 434                 result, request_config = self.RunCommit(commit_upto, brd,
 435                         work_dir, do_config, self.builder.config_only,
 436                         force_build or self.builder.force_build,
 437                         self.builder.force_build_failures)
 438                 failed = result.return_code or result.stderr
 439                 did_config = do_config
 440                 if failed and not do_config:
 441                     # If our incremental build failed, try building again
 442                     # with a reconfig.
 443                     if self.builder.force_config_on_failure:
 444                         result, request_config = self.RunCommit(commit_upto,
 445                             brd, work_dir, True, False, True, False)
 446                         did_config = True
 447                 if not self.builder.force_reconfig:
 448                     do_config = request_config
 449
 450                 # If we built that commit, then config is done. But if we got
 451                 # an warning, reconfig next time to force it to build the same
 452                 # files that created warnings this time. Otherwise an
 453                 # incremental build may not build the same file, and we will
 454                 # think that the warning has gone away.
 455                 # We could avoid this by using -Werror everywhere...
 456                 # For errors, the problem doesn't happen, since presumably
 457                 # the build stopped and didn't generate output, so will retry
 458                 # that file next time. So we could detect warnings and deal
 459                 # with them specially here. For now, we just reconfigure if
 460                 # anything goes work.
 461                 # Of course this is substantially slower if there are build
 462                 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
 463                 # have problems).
 464                 if (failed and not result.already_done and not did_config and
 465                         self.builder.force_config_on_failure):
 466                     # If this build failed, try the next one with a
 467                     # reconfigure.
 468                     # Sometimes if the board_config.h file changes it can mess
 469                     # with dependencies, and we get:
 470                     # make: *** No rule to make target `include/autoconf.mk',
 471                     #     needed by `depend'.
 472                     do_config = True
 473                     force_build = True
 474                 else:
 475                     force_build = False
 476                     if self.builder.force_config_on_failure:
 477                         if failed:
 478                             do_config = True
 479                     result.commit_upto = commit_upto
 480                     if result.return_code < 0:
 481                         raise ValueError('Interrupt')
 482
 483                 # We have the build results, so output the result
 484                 self._WriteResult(result, job.keep_outputs)
 485                 self.builder.out_queue.put(result)
 486         else:
 487             # Just build the currently checked-out build
 488             result, request_config = self.RunCommit(None, brd, work_dir, True,
 489                         self.builder.config_only, True,
 490                         self.builder.force_build_failures)
 491             result.commit_upto = 0
 492             self._WriteResult(result, job.keep_outputs)
 493             self.builder.out_queue.put(result)
 494
 495     def run(self):
 496         """Our thread's run function
 497
 498         This thread picks a job from the queue, runs it, and then goes to the
 499         next job.
 500         """
 501         while True:
 502             job = self.builder.queue.get()
 503             self.RunJob(job)
 504             self.builder.queue.task_done()