1 # Copyright (c) 2014 Google, Inc
3 # SPDX-License-Identifier: GPL-2.0+
15 RETURN_CODE_RETRY = -1
17 def Mkdir(dirname, parents = False):
18 """Make a directory if it doesn't already exist.
21 dirname: Directory to create
28 except OSError as err:
29 if err.errno == errno.EEXIST:
35 """Holds information about a job to be performed by a thread
38 board: Board object to build
39 commits: List of commit options to build.
46 class ResultThread(threading.Thread):
47 """This thread processes results from builder threads.
49 It simply passes the results on to the builder. There is only one
50 result thread, and this helps to serialise the build output.
52 def __init__(self, builder):
53 """Set up a new result thread
56 builder: Builder which will be sent each result
58 threading.Thread.__init__(self)
59 self.builder = builder
62 """Called to start up the result thread.
64 We collect the next result job and pass it on to the build.
67 result = self.builder.out_queue.get()
68 self.builder.ProcessResult(result)
69 self.builder.out_queue.task_done()
72 class BuilderThread(threading.Thread):
73 """This thread builds U-Boot for a particular board.
75 An input queue provides each new job. We run 'make' to build U-Boot
76 and then pass the results on to the output queue.
79 builder: The builder which contains information we might need
80 thread_num: Our thread number (0-n-1), used to decide on a
83 def __init__(self, builder, thread_num, incremental, per_board_out_dir):
84 """Set up a new builder thread"""
85 threading.Thread.__init__(self)
86 self.builder = builder
87 self.thread_num = thread_num
88 self.incremental = incremental
89 self.per_board_out_dir = per_board_out_dir
91 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
92 """Run 'make' on a particular commit and board.
94 The source code will already be checked out, so the 'commit'
95 argument is only for information.
98 commit: Commit object that is being built
99 brd: Board object that is being built
100 stage: Stage of the build. Valid stages are:
101 mrproper - can be called to clean source
102 config - called to configure for a board
103 build - the main make invocation - it does the build
104 args: A list of arguments to pass to 'make'
105 kwargs: A list of keyword arguments to pass to command.RunPipe()
110 return self.builder.do_make(commit, brd, stage, cwd, *args,
113 def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
114 force_build, force_build_failures):
115 """Build a particular commit.
117 If the build is already done, and we are not forcing a build, we skip
118 the build and just return the previously-saved results.
121 commit_upto: Commit number to build (0...n-1)
122 brd: Board object to build
123 work_dir: Directory to which the source will be checked out
124 do_config: True to run a make <board>_defconfig on the source
125 config_only: Only configure the source, do not build it
126 force_build: Force a build even if one was previously done
127 force_build_failures: Force a bulid if the previous result showed
132 - CommandResult object containing the results of the build
133 - boolean indicating whether 'make config' is still needed
135 # Create a default result - it will be overwritte by the call to
136 # self.Make() below, in the event that we do a build.
137 result = command.CommandResult()
138 result.return_code = 0
139 if self.builder.in_tree:
142 if self.per_board_out_dir:
143 out_rel_dir = os.path.join('..', brd.target)
145 out_rel_dir = 'build'
146 out_dir = os.path.join(work_dir, out_rel_dir)
148 # Check if the job was already completed last time
149 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
150 result.already_done = os.path.exists(done_file)
151 will_build = (force_build or force_build_failures or
152 not result.already_done)
153 if result.already_done:
154 # Get the return code from that build and use it
155 with open(done_file, 'r') as fd:
156 result.return_code = int(fd.readline())
158 # Check the signal that the build needs to be retried
159 if result.return_code == RETURN_CODE_RETRY:
162 err_file = self.builder.GetErrFile(commit_upto, brd.target)
163 if os.path.exists(err_file) and os.stat(err_file).st_size:
164 result.stderr = 'bad'
165 elif not force_build:
166 # The build passed, so no need to build it again
170 # We are going to have to build it. First, get a toolchain
171 if not self.toolchain:
173 self.toolchain = self.builder.toolchains.Select(brd.arch)
174 except ValueError as err:
175 result.return_code = 10
177 result.stderr = str(err)
178 # TODO(sjg@chromium.org): This gets swallowed, but needs
182 # Checkout the right commit
183 if self.builder.commits:
184 commit = self.builder.commits[commit_upto]
185 if self.builder.checkout:
186 git_dir = os.path.join(work_dir, '.git')
187 gitutil.Checkout(commit.hash, git_dir, work_dir,
192 # Set up the environment and command line
193 env = self.toolchain.MakeEnvironment(self.builder.full_path)
197 src_dir = os.path.realpath(work_dir)
198 if not self.builder.in_tree:
199 if commit_upto is None:
200 # In this case we are building in the original source
201 # directory (i.e. the current directory where buildman
202 # is invoked. The output directory is set to this
203 # thread's selected work directory.
205 # Symlinks can confuse U-Boot's Makefile since
206 # we may use '..' in our path, so remove them.
207 out_dir = os.path.realpath(out_dir)
208 args.append('O=%s' % out_dir)
210 src_dir = os.getcwd()
212 args.append('O=%s' % out_rel_dir)
213 if self.builder.verbose_build:
217 if self.builder.num_jobs is not None:
218 args.extend(['-j', str(self.builder.num_jobs)])
219 if self.builder.warnings_as_errors:
220 args.append('KCFLAGS=-Werror')
221 config_args = ['%s_defconfig' % brd.target]
223 args.extend(self.builder.toolchains.GetMakeArguments(brd))
225 # If we need to reconfigure, do that now
228 if not self.incremental:
229 result = self.Make(commit, brd, 'mrproper', cwd,
230 'mrproper', *args, env=env)
231 config_out += result.combined
232 result = self.Make(commit, brd, 'config', cwd,
233 *(args + config_args), env=env)
234 config_out += result.combined
235 do_config = False # No need to configure next time
236 if result.return_code == 0:
239 result = self.Make(commit, brd, 'build', cwd, *args,
241 result.stderr = result.stderr.replace(src_dir + '/', '')
242 if self.builder.verbose_build:
243 result.stdout = config_out + result.stdout
245 result.return_code = 1
246 result.stderr = 'No tool chain for %s\n' % brd.arch
247 result.already_done = False
249 result.toolchain = self.toolchain
251 result.commit_upto = commit_upto
252 result.out_dir = out_dir
253 return result, do_config
255 def _WriteResult(self, result, keep_outputs):
256 """Write a built result to the output directory.
259 result: CommandResult object containing result to write
260 keep_outputs: True to store the output binaries, False
264 if result.return_code < 0:
267 # If we think this might have been aborted with Ctrl-C, record the
268 # failure but not that we are 'done' with this board. A retry may fix
270 maybe_aborted = result.stderr and 'No child processes' in result.stderr
272 if result.already_done:
275 # Write the output and stderr
276 output_dir = self.builder._GetOutputDir(result.commit_upto)
278 build_dir = self.builder.GetBuildDir(result.commit_upto,
282 outfile = os.path.join(build_dir, 'log')
283 with open(outfile, 'w') as fd:
285 # We don't want unicode characters in log files
286 fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
288 errfile = self.builder.GetErrFile(result.commit_upto,
291 with open(errfile, 'w') as fd:
292 # We don't want unicode characters in log files
293 fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
294 elif os.path.exists(errfile):
298 # Write the build result and toolchain information.
299 done_file = self.builder.GetDoneFile(result.commit_upto,
301 with open(done_file, 'w') as fd:
303 # Special code to indicate we need to retry
304 fd.write('%s' % RETURN_CODE_RETRY)
306 fd.write('%s' % result.return_code)
307 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
308 print >>fd, 'gcc', result.toolchain.gcc
309 print >>fd, 'path', result.toolchain.path
310 print >>fd, 'cross', result.toolchain.cross
311 print >>fd, 'arch', result.toolchain.arch
312 fd.write('%s' % result.return_code)
314 # Write out the image and function size information and an objdump
315 env = result.toolchain.MakeEnvironment(self.builder.full_path)
317 for fname in ['u-boot', 'spl/u-boot-spl']:
318 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
319 nm_result = command.RunPipe([cmd], capture=True,
320 capture_stderr=True, cwd=result.out_dir,
321 raise_on_error=False, env=env)
323 nm = self.builder.GetFuncSizesFile(result.commit_upto,
324 result.brd.target, fname)
325 with open(nm, 'w') as fd:
326 print >>fd, nm_result.stdout,
328 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
329 dump_result = command.RunPipe([cmd], capture=True,
330 capture_stderr=True, cwd=result.out_dir,
331 raise_on_error=False, env=env)
333 if dump_result.stdout:
334 objdump = self.builder.GetObjdumpFile(result.commit_upto,
335 result.brd.target, fname)
336 with open(objdump, 'w') as fd:
337 print >>fd, dump_result.stdout,
338 for line in dump_result.stdout.splitlines():
339 fields = line.split()
340 if len(fields) > 5 and fields[1] == '.rodata':
341 rodata_size = fields[2]
343 cmd = ['%ssize' % self.toolchain.cross, fname]
344 size_result = command.RunPipe([cmd], capture=True,
345 capture_stderr=True, cwd=result.out_dir,
346 raise_on_error=False, env=env)
347 if size_result.stdout:
348 lines.append(size_result.stdout.splitlines()[1] + ' ' +
351 # Write out the image sizes file. This is similar to the output
352 # of binutil's 'size' utility, but it omits the header line and
353 # adds an additional hex value at the end of each line for the
356 sizes = self.builder.GetSizesFile(result.commit_upto,
358 with open(sizes, 'w') as fd:
359 print >>fd, '\n'.join(lines)
361 # Write out the configuration files, with a special case for SPL
362 for dirname in ['', 'spl', 'tpl']:
363 self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
364 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
365 'include/autoconf.mk', 'include/generated/autoconf.h'])
367 # Now write the actual build output
369 self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
370 '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
373 def CopyFiles(self, out_dir, build_dir, dirname, patterns):
374 """Copy files from the build directory to the output.
377 out_dir: Path to output directory containing the files
378 build_dir: Place to copy the files
379 dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
380 patterns: A list of filenames (strings) to copy, each relative
381 to the build directory
383 for pattern in patterns:
384 file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
385 for fname in file_list:
386 target = os.path.basename(fname)
388 base, ext = os.path.splitext(target)
390 target = '%s-%s%s' % (base, dirname, ext)
391 shutil.copy(fname, os.path.join(build_dir, target))
393 def RunJob(self, job):
396 A job consists of a building a list of commits for a particular board.
402 work_dir = self.builder.GetThreadDir(self.thread_num)
403 self.toolchain = None
405 # Run 'make board_defconfig' on the first commit
409 for commit_upto in range(0, len(job.commits), job.step):
410 result, request_config = self.RunCommit(commit_upto, brd,
411 work_dir, do_config, self.builder.config_only,
412 force_build or self.builder.force_build,
413 self.builder.force_build_failures)
414 failed = result.return_code or result.stderr
415 did_config = do_config
416 if failed and not do_config:
417 # If our incremental build failed, try building again
419 if self.builder.force_config_on_failure:
420 result, request_config = self.RunCommit(commit_upto,
421 brd, work_dir, True, False, True, False)
423 if not self.builder.force_reconfig:
424 do_config = request_config
426 # If we built that commit, then config is done. But if we got
427 # an warning, reconfig next time to force it to build the same
428 # files that created warnings this time. Otherwise an
429 # incremental build may not build the same file, and we will
430 # think that the warning has gone away.
431 # We could avoid this by using -Werror everywhere...
432 # For errors, the problem doesn't happen, since presumably
433 # the build stopped and didn't generate output, so will retry
434 # that file next time. So we could detect warnings and deal
435 # with them specially here. For now, we just reconfigure if
436 # anything goes work.
437 # Of course this is substantially slower if there are build
438 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
440 if (failed and not result.already_done and not did_config and
441 self.builder.force_config_on_failure):
442 # If this build failed, try the next one with a
444 # Sometimes if the board_config.h file changes it can mess
445 # with dependencies, and we get:
446 # make: *** No rule to make target `include/autoconf.mk',
447 # needed by `depend'.
452 if self.builder.force_config_on_failure:
455 result.commit_upto = commit_upto
456 if result.return_code < 0:
457 raise ValueError('Interrupt')
459 # We have the build results, so output the result
460 self._WriteResult(result, job.keep_outputs)
461 self.builder.out_queue.put(result)
463 # Just build the currently checked-out build
464 result, request_config = self.RunCommit(None, brd, work_dir, True,
465 self.builder.config_only, True,
466 self.builder.force_build_failures)
467 result.commit_upto = 0
468 self._WriteResult(result, job.keep_outputs)
469 self.builder.out_queue.put(result)
472 """Our thread's run function
474 This thread picks a job from the queue, runs it, and then goes to the
478 job = self.builder.queue.get()
480 self.builder.queue.task_done()