tools/buildman/builderthread.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4
   5 import errno
   6 import glob
   7 import os
   8 import shutil
   9 import sys
  10 import threading
  11
  12 import command
  13 import gitutil
  14
  15 RETURN_CODE_RETRY = -1
  16
  17 def Mkdir(dirname, parents = False):
  18     """Make a directory if it doesn't already exist.
  19
  20     Args:
  21         dirname: Directory to create
  22     """
  23     try:
  24         if parents:
  25             os.makedirs(dirname)
  26         else:
  27             os.mkdir(dirname)
  28     except OSError as err:
  29         if err.errno == errno.EEXIST:
  30             if os.path.realpath('.') == os.path.realpath(dirname):
  31                 print "Cannot create the current working directory '%s'!" % dirname
  32                 sys.exit(1)
  33             pass
  34         else:
  35             raise
  36
  37 class BuilderJob:
  38     """Holds information about a job to be performed by a thread
  39
  40     Members:
  41         board: Board object to build
  42         commits: List of commit options to build.
  43     """
  44     def __init__(self):
  45         self.board = None
  46         self.commits = []
  47
  48
  49 class ResultThread(threading.Thread):
  50     """This thread processes results from builder threads.
  51
  52     It simply passes the results on to the builder. There is only one
  53     result thread, and this helps to serialise the build output.
  54     """
  55     def __init__(self, builder):
  56         """Set up a new result thread
  57
  58         Args:
  59             builder: Builder which will be sent each result
  60         """
  61         threading.Thread.__init__(self)
  62         self.builder = builder
  63
  64     def run(self):
  65         """Called to start up the result thread.
  66
  67         We collect the next result job and pass it on to the build.
  68         """
  69         while True:
  70             result = self.builder.out_queue.get()
  71             self.builder.ProcessResult(result)
  72             self.builder.out_queue.task_done()
  73
  74
  75 class BuilderThread(threading.Thread):
  76     """This thread builds U-Boot for a particular board.
  77
  78     An input queue provides each new job. We run 'make' to build U-Boot
  79     and then pass the results on to the output queue.
  80
  81     Members:
  82         builder: The builder which contains information we might need
  83         thread_num: Our thread number (0-n-1), used to decide on a
  84                 temporary directory
  85     """
  86     def __init__(self, builder, thread_num, incremental, per_board_out_dir):
  87         """Set up a new builder thread"""
  88         threading.Thread.__init__(self)
  89         self.builder = builder
  90         self.thread_num = thread_num
  91         self.incremental = incremental
  92         self.per_board_out_dir = per_board_out_dir
  93
  94     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
  95         """Run 'make' on a particular commit and board.
  96
  97         The source code will already be checked out, so the 'commit'
  98         argument is only for information.
  99
 100         Args:
 101             commit: Commit object that is being built
 102             brd: Board object that is being built
 103             stage: Stage of the build. Valid stages are:
 104                         mrproper - can be called to clean source
 105                         config - called to configure for a board
 106                         build - the main make invocation - it does the build
 107             args: A list of arguments to pass to 'make'
 108             kwargs: A list of keyword arguments to pass to command.RunPipe()
 109
 110         Returns:
 111             CommandResult object
 112         """
 113         return self.builder.do_make(commit, brd, stage, cwd, *args,
 114                 **kwargs)
 115
 116     def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
 117                   force_build, force_build_failures):
 118         """Build a particular commit.
 119
 120         If the build is already done, and we are not forcing a build, we skip
 121         the build and just return the previously-saved results.
 122
 123         Args:
 124             commit_upto: Commit number to build (0...n-1)
 125             brd: Board object to build
 126             work_dir: Directory to which the source will be checked out
 127             do_config: True to run a make <board>_defconfig on the source
 128             config_only: Only configure the source, do not build it
 129             force_build: Force a build even if one was previously done
 130             force_build_failures: Force a bulid if the previous result showed
 131                 failure
 132
 133         Returns:
 134             tuple containing:
 135                 - CommandResult object containing the results of the build
 136                 - boolean indicating whether 'make config' is still needed
 137         """
 138         # Create a default result - it will be overwritte by the call to
 139         # self.Make() below, in the event that we do a build.
 140         result = command.CommandResult()
 141         result.return_code = 0
 142         if self.builder.in_tree:
 143             out_dir = work_dir
 144         else:
 145             if self.per_board_out_dir:
 146                 out_rel_dir = os.path.join('..', brd.target)
 147             else:
 148                 out_rel_dir = 'build'
 149             out_dir = os.path.join(work_dir, out_rel_dir)
 150
 151         # Check if the job was already completed last time
 152         done_file = self.builder.GetDoneFile(commit_upto, brd.target)
 153         result.already_done = os.path.exists(done_file)
 154         will_build = (force_build or force_build_failures or
 155             not result.already_done)
 156         if result.already_done:
 157             # Get the return code from that build and use it
 158             with open(done_file, 'r') as fd:
 159                 try:
 160                     result.return_code = int(fd.readline())
 161                 except ValueError:
 162                     # The file may be empty due to running out of disk space.
 163                     # Try a rebuild
 164                     result.return_code = RETURN_CODE_RETRY
 165
 166             # Check the signal that the build needs to be retried
 167             if result.return_code == RETURN_CODE_RETRY:
 168                 will_build = True
 169             elif will_build:
 170                 err_file = self.builder.GetErrFile(commit_upto, brd.target)
 171                 if os.path.exists(err_file) and os.stat(err_file).st_size:
 172                     result.stderr = 'bad'
 173                 elif not force_build:
 174                     # The build passed, so no need to build it again
 175                     will_build = False
 176
 177         if will_build:
 178             # We are going to have to build it. First, get a toolchain
 179             if not self.toolchain:
 180                 try:
 181                     self.toolchain = self.builder.toolchains.Select(brd.arch)
 182                 except ValueError as err:
 183                     result.return_code = 10
 184                     result.stdout = ''
 185                     result.stderr = str(err)
 186                     # TODO(sjg@chromium.org): This gets swallowed, but needs
 187                     # to be reported.
 188
 189             if self.toolchain:
 190                 # Checkout the right commit
 191                 if self.builder.commits:
 192                     commit = self.builder.commits[commit_upto]
 193                     if self.builder.checkout:
 194                         git_dir = os.path.join(work_dir, '.git')
 195                         gitutil.Checkout(commit.hash, git_dir, work_dir,
 196                                          force=True)
 197                 else:
 198                     commit = 'current'
 199
 200                 # Set up the environment and command line
 201                 env = self.toolchain.MakeEnvironment(self.builder.full_path)
 202                 Mkdir(out_dir)
 203                 args = []
 204                 cwd = work_dir
 205                 src_dir = os.path.realpath(work_dir)
 206                 if not self.builder.in_tree:
 207                     if commit_upto is None:
 208                         # In this case we are building in the original source
 209                         # directory (i.e. the current directory where buildman
 210                         # is invoked. The output directory is set to this
 211                         # thread's selected work directory.
 212                         #
 213                         # Symlinks can confuse U-Boot's Makefile since
 214                         # we may use '..' in our path, so remove them.
 215                         out_dir = os.path.realpath(out_dir)
 216                         args.append('O=%s' % out_dir)
 217                         cwd = None
 218                         src_dir = os.getcwd()
 219                     else:
 220                         args.append('O=%s' % out_rel_dir)
 221                 if self.builder.verbose_build:
 222                     args.append('V=1')
 223                 else:
 224                     args.append('-s')
 225                 if self.builder.num_jobs is not None:
 226                     args.extend(['-j', str(self.builder.num_jobs)])
 227                 if self.builder.warnings_as_errors:
 228                     args.append('KCFLAGS=-Werror')
 229                 config_args = ['%s_defconfig' % brd.target]
 230                 config_out = ''
 231                 args.extend(self.builder.toolchains.GetMakeArguments(brd))
 232                 args.extend(self.toolchain.MakeArgs())
 233
 234                 # If we need to reconfigure, do that now
 235                 if do_config:
 236                     config_out = ''
 237                     if not self.incremental:
 238                         result = self.Make(commit, brd, 'mrproper', cwd,
 239                                 'mrproper', *args, env=env)
 240                         config_out += result.combined
 241                     result = self.Make(commit, brd, 'config', cwd,
 242                             *(args + config_args), env=env)
 243                     config_out += result.combined
 244                     do_config = False   # No need to configure next time
 245                 if result.return_code == 0:
 246                     if config_only:
 247                         args.append('cfg')
 248                     result = self.Make(commit, brd, 'build', cwd, *args,
 249                             env=env)
 250                 result.stderr = result.stderr.replace(src_dir + '/', '')
 251                 if self.builder.verbose_build:
 252                     result.stdout = config_out + result.stdout
 253             else:
 254                 result.return_code = 1
 255                 result.stderr = 'No tool chain for %s\n' % brd.arch
 256             result.already_done = False
 257
 258         result.toolchain = self.toolchain
 259         result.brd = brd
 260         result.commit_upto = commit_upto
 261         result.out_dir = out_dir
 262         return result, do_config
 263
 264     def _WriteResult(self, result, keep_outputs):
 265         """Write a built result to the output directory.
 266
 267         Args:
 268             result: CommandResult object containing result to write
 269             keep_outputs: True to store the output binaries, False
 270                 to delete them
 271         """
 272         # Fatal error
 273         if result.return_code < 0:
 274             return
 275
 276         # If we think this might have been aborted with Ctrl-C, record the
 277         # failure but not that we are 'done' with this board. A retry may fix
 278         # it.
 279         maybe_aborted =  result.stderr and 'No child processes' in result.stderr
 280
 281         if result.already_done:
 282             return
 283
 284         # Write the output and stderr
 285         output_dir = self.builder._GetOutputDir(result.commit_upto)
 286         Mkdir(output_dir)
 287         build_dir = self.builder.GetBuildDir(result.commit_upto,
 288                 result.brd.target)
 289         Mkdir(build_dir)
 290
 291         outfile = os.path.join(build_dir, 'log')
 292         with open(outfile, 'w') as fd:
 293             if result.stdout:
 294                 # We don't want unicode characters in log files
 295                 fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
 296
 297         errfile = self.builder.GetErrFile(result.commit_upto,
 298                 result.brd.target)
 299         if result.stderr:
 300             with open(errfile, 'w') as fd:
 301                 # We don't want unicode characters in log files
 302                 fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
 303         elif os.path.exists(errfile):
 304             os.remove(errfile)
 305
 306         if result.toolchain:
 307             # Write the build result and toolchain information.
 308             done_file = self.builder.GetDoneFile(result.commit_upto,
 309                     result.brd.target)
 310             with open(done_file, 'w') as fd:
 311                 if maybe_aborted:
 312                     # Special code to indicate we need to retry
 313                     fd.write('%s' % RETURN_CODE_RETRY)
 314                 else:
 315                     fd.write('%s' % result.return_code)
 316             with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
 317                 print >>fd, 'gcc', result.toolchain.gcc
 318                 print >>fd, 'path', result.toolchain.path
 319                 print >>fd, 'cross', result.toolchain.cross
 320                 print >>fd, 'arch', result.toolchain.arch
 321                 fd.write('%s' % result.return_code)
 322
 323             # Write out the image and function size information and an objdump
 324             env = result.toolchain.MakeEnvironment(self.builder.full_path)
 325             with open(os.path.join(build_dir, 'env'), 'w') as fd:
 326                 for var in sorted(env.keys()):
 327                     print >>fd, '%s="%s"' % (var, env[var])
 328             lines = []
 329             for fname in ['u-boot', 'spl/u-boot-spl']:
 330                 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
 331                 nm_result = command.RunPipe([cmd], capture=True,
 332                         capture_stderr=True, cwd=result.out_dir,
 333                         raise_on_error=False, env=env)
 334                 if nm_result.stdout:
 335                     nm = self.builder.GetFuncSizesFile(result.commit_upto,
 336                                     result.brd.target, fname)
 337                     with open(nm, 'w') as fd:
 338                         print >>fd, nm_result.stdout,
 339
 340                 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
 341                 dump_result = command.RunPipe([cmd], capture=True,
 342                         capture_stderr=True, cwd=result.out_dir,
 343                         raise_on_error=False, env=env)
 344                 rodata_size = ''
 345                 if dump_result.stdout:
 346                     objdump = self.builder.GetObjdumpFile(result.commit_upto,
 347                                     result.brd.target, fname)
 348                     with open(objdump, 'w') as fd:
 349                         print >>fd, dump_result.stdout,
 350                     for line in dump_result.stdout.splitlines():
 351                         fields = line.split()
 352                         if len(fields) > 5 and fields[1] == '.rodata':
 353                             rodata_size = fields[2]
 354
 355                 cmd = ['%ssize' % self.toolchain.cross, fname]
 356                 size_result = command.RunPipe([cmd], capture=True,
 357                         capture_stderr=True, cwd=result.out_dir,
 358                         raise_on_error=False, env=env)
 359                 if size_result.stdout:
 360                     lines.append(size_result.stdout.splitlines()[1] + ' ' +
 361                                  rodata_size)
 362
 363             # Extract the environment from U-Boot and dump it out
 364             cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary',
 365                    '-j', '.rodata.default_environment',
 366                    'env/built-in.o', 'uboot.env']
 367             command.RunPipe([cmd], capture=True,
 368                             capture_stderr=True, cwd=result.out_dir,
 369                             raise_on_error=False, env=env)
 370             ubootenv = os.path.join(result.out_dir, 'uboot.env')
 371             self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
 372
 373             # Write out the image sizes file. This is similar to the output
 374             # of binutil's 'size' utility, but it omits the header line and
 375             # adds an additional hex value at the end of each line for the
 376             # rodata size
 377             if len(lines):
 378                 sizes = self.builder.GetSizesFile(result.commit_upto,
 379                                 result.brd.target)
 380                 with open(sizes, 'w') as fd:
 381                     print >>fd, '\n'.join(lines)
 382
 383         # Write out the configuration files, with a special case for SPL
 384         for dirname in ['', 'spl', 'tpl']:
 385             self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
 386                 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
 387                 'include/autoconf.mk', 'include/generated/autoconf.h'])
 388
 389         # Now write the actual build output
 390         if keep_outputs:
 391             self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
 392                 '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
 393                 'spl/u-boot-spl*'])
 394
 395     def CopyFiles(self, out_dir, build_dir, dirname, patterns):
 396         """Copy files from the build directory to the output.
 397
 398         Args:
 399             out_dir: Path to output directory containing the files
 400             build_dir: Place to copy the files
 401             dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
 402             patterns: A list of filenames (strings) to copy, each relative
 403                to the build directory
 404         """
 405         for pattern in patterns:
 406             file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
 407             for fname in file_list:
 408                 target = os.path.basename(fname)
 409                 if dirname:
 410                     base, ext = os.path.splitext(target)
 411                     if ext:
 412                         target = '%s-%s%s' % (base, dirname, ext)
 413                 shutil.copy(fname, os.path.join(build_dir, target))
 414
 415     def RunJob(self, job):
 416         """Run a single job
 417
 418         A job consists of a building a list of commits for a particular board.
 419
 420         Args:
 421             job: Job to build
 422         """
 423         brd = job.board
 424         work_dir = self.builder.GetThreadDir(self.thread_num)
 425         self.toolchain = None
 426         if job.commits:
 427             # Run 'make board_defconfig' on the first commit
 428             do_config = True
 429             commit_upto  = 0
 430             force_build = False
 431             for commit_upto in range(0, len(job.commits), job.step):
 432                 result, request_config = self.RunCommit(commit_upto, brd,
 433                         work_dir, do_config, self.builder.config_only,
 434                         force_build or self.builder.force_build,
 435                         self.builder.force_build_failures)
 436                 failed = result.return_code or result.stderr
 437                 did_config = do_config
 438                 if failed and not do_config:
 439                     # If our incremental build failed, try building again
 440                     # with a reconfig.
 441                     if self.builder.force_config_on_failure:
 442                         result, request_config = self.RunCommit(commit_upto,
 443                             brd, work_dir, True, False, True, False)
 444                         did_config = True
 445                 if not self.builder.force_reconfig:
 446                     do_config = request_config
 447
 448                 # If we built that commit, then config is done. But if we got
 449                 # an warning, reconfig next time to force it to build the same
 450                 # files that created warnings this time. Otherwise an
 451                 # incremental build may not build the same file, and we will
 452                 # think that the warning has gone away.
 453                 # We could avoid this by using -Werror everywhere...
 454                 # For errors, the problem doesn't happen, since presumably
 455                 # the build stopped and didn't generate output, so will retry
 456                 # that file next time. So we could detect warnings and deal
 457                 # with them specially here. For now, we just reconfigure if
 458                 # anything goes work.
 459                 # Of course this is substantially slower if there are build
 460                 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
 461                 # have problems).
 462                 if (failed and not result.already_done and not did_config and
 463                         self.builder.force_config_on_failure):
 464                     # If this build failed, try the next one with a
 465                     # reconfigure.
 466                     # Sometimes if the board_config.h file changes it can mess
 467                     # with dependencies, and we get:
 468                     # make: *** No rule to make target `include/autoconf.mk',
 469                     #     needed by `depend'.
 470                     do_config = True
 471                     force_build = True
 472                 else:
 473                     force_build = False
 474                     if self.builder.force_config_on_failure:
 475                         if failed:
 476                             do_config = True
 477                     result.commit_upto = commit_upto
 478                     if result.return_code < 0:
 479                         raise ValueError('Interrupt')
 480
 481                 # We have the build results, so output the result
 482                 self._WriteResult(result, job.keep_outputs)
 483                 self.builder.out_queue.put(result)
 484         else:
 485             # Just build the currently checked-out build
 486             result, request_config = self.RunCommit(None, brd, work_dir, True,
 487                         self.builder.config_only, True,
 488                         self.builder.force_build_failures)
 489             result.commit_upto = 0
 490             self._WriteResult(result, job.keep_outputs)
 491             self.builder.out_queue.put(result)
 492
 493     def run(self):
 494         """Our thread's run function
 495
 496         This thread picks a job from the queue, runs it, and then goes to the
 497         next job.
 498         """
 499         while True:
 500             job = self.builder.queue.get()
 501             self.RunJob(job)
 502             self.builder.queue.task_done()