1 # Copyright (c) 2014 Google, Inc
3 # SPDX-License-Identifier: GPL-2.0+
16 """Make a directory if it doesn't already exist.
19 dirname: Directory to create
23 except OSError as err:
24 if err.errno == errno.EEXIST:
30 """Holds information about a job to be performed by a thread
33 board: Board object to build
34 commits: List of commit options to build.
41 class ResultThread(threading.Thread):
42 """This thread processes results from builder threads.
44 It simply passes the results on to the builder. There is only one
45 result thread, and this helps to serialise the build output.
47 def __init__(self, builder):
48 """Set up a new result thread
51 builder: Builder which will be sent each result
53 threading.Thread.__init__(self)
54 self.builder = builder
57 """Called to start up the result thread.
59 We collect the next result job and pass it on to the build.
62 result = self.builder.out_queue.get()
63 self.builder.ProcessResult(result)
64 self.builder.out_queue.task_done()
67 class BuilderThread(threading.Thread):
68 """This thread builds U-Boot for a particular board.
70 An input queue provides each new job. We run 'make' to build U-Boot
71 and then pass the results on to the output queue.
74 builder: The builder which contains information we might need
75 thread_num: Our thread number (0-n-1), used to decide on a
78 def __init__(self, builder, thread_num):
79 """Set up a new builder thread"""
80 threading.Thread.__init__(self)
81 self.builder = builder
82 self.thread_num = thread_num
84 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
85 """Run 'make' on a particular commit and board.
87 The source code will already be checked out, so the 'commit'
88 argument is only for information.
91 commit: Commit object that is being built
92 brd: Board object that is being built
93 stage: Stage of the build. Valid stages are:
94 mrproper - can be called to clean source
95 config - called to configure for a board
96 build - the main make invocation - it does the build
97 args: A list of arguments to pass to 'make'
98 kwargs: A list of keyword arguments to pass to command.RunPipe()
103 return self.builder.do_make(commit, brd, stage, cwd, *args,
106 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
107 force_build_failures):
108 """Build a particular commit.
110 If the build is already done, and we are not forcing a build, we skip
111 the build and just return the previously-saved results.
114 commit_upto: Commit number to build (0...n-1)
115 brd: Board object to build
116 work_dir: Directory to which the source will be checked out
117 do_config: True to run a make <board>_defconfig on the source
118 force_build: Force a build even if one was previously done
119 force_build_failures: Force a bulid if the previous result showed
124 - CommandResult object containing the results of the build
125 - boolean indicating whether 'make config' is still needed
127 # Create a default result - it will be overwritte by the call to
128 # self.Make() below, in the event that we do a build.
129 result = command.CommandResult()
130 result.return_code = 0
131 if self.builder.in_tree:
134 out_dir = os.path.join(work_dir, 'build')
136 # Check if the job was already completed last time
137 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
138 result.already_done = os.path.exists(done_file)
139 will_build = (force_build or force_build_failures or
140 not result.already_done)
141 if result.already_done:
142 # Get the return code from that build and use it
143 with open(done_file, 'r') as fd:
144 result.return_code = int(fd.readline())
146 err_file = self.builder.GetErrFile(commit_upto, brd.target)
147 if os.path.exists(err_file) and os.stat(err_file).st_size:
148 result.stderr = 'bad'
149 elif not force_build:
150 # The build passed, so no need to build it again
154 # We are going to have to build it. First, get a toolchain
155 if not self.toolchain:
157 self.toolchain = self.builder.toolchains.Select(brd.arch)
158 except ValueError as err:
159 result.return_code = 10
161 result.stderr = str(err)
162 # TODO(sjg@chromium.org): This gets swallowed, but needs
166 # Checkout the right commit
167 if self.builder.commits:
168 commit = self.builder.commits[commit_upto]
169 if self.builder.checkout:
170 git_dir = os.path.join(work_dir, '.git')
171 gitutil.Checkout(commit.hash, git_dir, work_dir,
176 # Set up the environment and command line
177 env = self.toolchain.MakeEnvironment()
181 src_dir = os.path.realpath(work_dir)
182 if not self.builder.in_tree:
183 if commit_upto is None:
184 # In this case we are building in the original source
185 # directory (i.e. the current directory where buildman
186 # is invoked. The output directory is set to this
187 # thread's selected work directory.
189 # Symlinks can confuse U-Boot's Makefile since
190 # we may use '..' in our path, so remove them.
191 work_dir = os.path.realpath(work_dir)
192 args.append('O=%s/build' % work_dir)
194 src_dir = os.getcwd()
196 args.append('O=build')
198 if self.builder.num_jobs is not None:
199 args.extend(['-j', str(self.builder.num_jobs)])
200 config_args = ['%s_defconfig' % brd.target]
202 args.extend(self.builder.toolchains.GetMakeArguments(brd))
204 # If we need to reconfigure, do that now
206 result = self.Make(commit, brd, 'mrproper', cwd,
207 'mrproper', *args, env=env)
208 result = self.Make(commit, brd, 'config', cwd,
209 *(args + config_args), env=env)
210 config_out = result.combined
211 do_config = False # No need to configure next time
212 if result.return_code == 0:
213 result = self.Make(commit, brd, 'build', cwd, *args,
215 result.stderr = result.stderr.replace(src_dir + '/', '')
217 result.return_code = 1
218 result.stderr = 'No tool chain for %s\n' % brd.arch
219 result.already_done = False
221 result.toolchain = self.toolchain
223 result.commit_upto = commit_upto
224 result.out_dir = out_dir
225 return result, do_config
227 def _WriteResult(self, result, keep_outputs):
228 """Write a built result to the output directory.
231 result: CommandResult object containing result to write
232 keep_outputs: True to store the output binaries, False
236 if result.return_code < 0:
240 if result.stderr and 'No child processes' in result.stderr:
243 if result.already_done:
246 # Write the output and stderr
247 output_dir = self.builder._GetOutputDir(result.commit_upto)
249 build_dir = self.builder.GetBuildDir(result.commit_upto,
253 outfile = os.path.join(build_dir, 'log')
254 with open(outfile, 'w') as fd:
256 fd.write(result.stdout)
258 errfile = self.builder.GetErrFile(result.commit_upto,
261 with open(errfile, 'w') as fd:
262 fd.write(result.stderr)
263 elif os.path.exists(errfile):
267 # Write the build result and toolchain information.
268 done_file = self.builder.GetDoneFile(result.commit_upto,
270 with open(done_file, 'w') as fd:
271 fd.write('%s' % result.return_code)
272 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
273 print >>fd, 'gcc', result.toolchain.gcc
274 print >>fd, 'path', result.toolchain.path
275 print >>fd, 'cross', result.toolchain.cross
276 print >>fd, 'arch', result.toolchain.arch
277 fd.write('%s' % result.return_code)
279 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
280 print >>fd, 'gcc', result.toolchain.gcc
281 print >>fd, 'path', result.toolchain.path
283 # Write out the image and function size information and an objdump
284 env = result.toolchain.MakeEnvironment()
286 for fname in ['u-boot', 'spl/u-boot-spl']:
287 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
288 nm_result = command.RunPipe([cmd], capture=True,
289 capture_stderr=True, cwd=result.out_dir,
290 raise_on_error=False, env=env)
292 nm = self.builder.GetFuncSizesFile(result.commit_upto,
293 result.brd.target, fname)
294 with open(nm, 'w') as fd:
295 print >>fd, nm_result.stdout,
297 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
298 dump_result = command.RunPipe([cmd], capture=True,
299 capture_stderr=True, cwd=result.out_dir,
300 raise_on_error=False, env=env)
302 if dump_result.stdout:
303 objdump = self.builder.GetObjdumpFile(result.commit_upto,
304 result.brd.target, fname)
305 with open(objdump, 'w') as fd:
306 print >>fd, dump_result.stdout,
307 for line in dump_result.stdout.splitlines():
308 fields = line.split()
309 if len(fields) > 5 and fields[1] == '.rodata':
310 rodata_size = fields[2]
312 cmd = ['%ssize' % self.toolchain.cross, fname]
313 size_result = command.RunPipe([cmd], capture=True,
314 capture_stderr=True, cwd=result.out_dir,
315 raise_on_error=False, env=env)
316 if size_result.stdout:
317 lines.append(size_result.stdout.splitlines()[1] + ' ' +
320 # Write out the image sizes file. This is similar to the output
321 # of binutil's 'size' utility, but it omits the header line and
322 # adds an additional hex value at the end of each line for the
325 sizes = self.builder.GetSizesFile(result.commit_upto,
327 with open(sizes, 'w') as fd:
328 print >>fd, '\n'.join(lines)
330 # Now write the actual build output
332 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
333 'include/autoconf.mk', 'spl/u-boot-spl',
334 'spl/u-boot-spl.bin']
335 for pattern in patterns:
336 file_list = glob.glob(os.path.join(result.out_dir, pattern))
337 for fname in file_list:
338 shutil.copy(fname, build_dir)
341 def RunJob(self, job):
344 A job consists of a building a list of commits for a particular board.
350 work_dir = self.builder.GetThreadDir(self.thread_num)
351 self.toolchain = None
353 # Run 'make board_defconfig' on the first commit
357 for commit_upto in range(0, len(job.commits), job.step):
358 result, request_config = self.RunCommit(commit_upto, brd,
360 force_build or self.builder.force_build,
361 self.builder.force_build_failures)
362 failed = result.return_code or result.stderr
363 did_config = do_config
364 if failed and not do_config:
365 # If our incremental build failed, try building again
367 if self.builder.force_config_on_failure:
368 result, request_config = self.RunCommit(commit_upto,
369 brd, work_dir, True, True, False)
371 if not self.builder.force_reconfig:
372 do_config = request_config
374 # If we built that commit, then config is done. But if we got
375 # an warning, reconfig next time to force it to build the same
376 # files that created warnings this time. Otherwise an
377 # incremental build may not build the same file, and we will
378 # think that the warning has gone away.
379 # We could avoid this by using -Werror everywhere...
380 # For errors, the problem doesn't happen, since presumably
381 # the build stopped and didn't generate output, so will retry
382 # that file next time. So we could detect warnings and deal
383 # with them specially here. For now, we just reconfigure if
384 # anything goes work.
385 # Of course this is substantially slower if there are build
386 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
388 if (failed and not result.already_done and not did_config and
389 self.builder.force_config_on_failure):
390 # If this build failed, try the next one with a
392 # Sometimes if the board_config.h file changes it can mess
393 # with dependencies, and we get:
394 # make: *** No rule to make target `include/autoconf.mk',
395 # needed by `depend'.
400 if self.builder.force_config_on_failure:
403 result.commit_upto = commit_upto
404 if result.return_code < 0:
405 raise ValueError('Interrupt')
407 # We have the build results, so output the result
408 self._WriteResult(result, job.keep_outputs)
409 self.builder.out_queue.put(result)
411 # Just build the currently checked-out build
412 result, request_config = self.RunCommit(None, brd, work_dir, True,
413 True, self.builder.force_build_failures)
414 result.commit_upto = 0
415 self._WriteResult(result, job.keep_outputs)
416 self.builder.out_queue.put(result)
419 """Our thread's run function
421 This thread picks a job from the queue, runs it, and then goes to the
426 job = self.builder.queue.get()
427 if self.builder.active and alive:
431 if self.builder.active and alive:
433 except Exception as err:
437 self.builder.queue.task_done()