1 # Copyright (c) 2014 Google, Inc
3 # SPDX-License-Identifier: GPL-2.0+
16 """Make a directory if it doesn't already exist.
19 dirname: Directory to create
23 except OSError as err:
24 if err.errno == errno.EEXIST:
30 """Holds information about a job to be performed by a thread
33 board: Board object to build
34 commits: List of commit options to build.
41 class ResultThread(threading.Thread):
42 """This thread processes results from builder threads.
44 It simply passes the results on to the builder. There is only one
45 result thread, and this helps to serialise the build output.
47 def __init__(self, builder):
48 """Set up a new result thread
51 builder: Builder which will be sent each result
53 threading.Thread.__init__(self)
54 self.builder = builder
57 """Called to start up the result thread.
59 We collect the next result job and pass it on to the build.
62 result = self.builder.out_queue.get()
63 self.builder.ProcessResult(result)
64 self.builder.out_queue.task_done()
67 class BuilderThread(threading.Thread):
68 """This thread builds U-Boot for a particular board.
70 An input queue provides each new job. We run 'make' to build U-Boot
71 and then pass the results on to the output queue.
74 builder: The builder which contains information we might need
75 thread_num: Our thread number (0-n-1), used to decide on a
78 def __init__(self, builder, thread_num):
79 """Set up a new builder thread"""
80 threading.Thread.__init__(self)
81 self.builder = builder
82 self.thread_num = thread_num
84 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
85 """Run 'make' on a particular commit and board.
87 The source code will already be checked out, so the 'commit'
88 argument is only for information.
91 commit: Commit object that is being built
92 brd: Board object that is being built
93 stage: Stage of the build. Valid stages are:
94 mrproper - can be called to clean source
95 config - called to configure for a board
96 build - the main make invocation - it does the build
97 args: A list of arguments to pass to 'make'
98 kwargs: A list of keyword arguments to pass to command.RunPipe()
103 return self.builder.do_make(commit, brd, stage, cwd, *args,
106 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
107 force_build_failures):
108 """Build a particular commit.
110 If the build is already done, and we are not forcing a build, we skip
111 the build and just return the previously-saved results.
114 commit_upto: Commit number to build (0...n-1)
115 brd: Board object to build
116 work_dir: Directory to which the source will be checked out
117 do_config: True to run a make <board>_defconfig on the source
118 force_build: Force a build even if one was previously done
119 force_build_failures: Force a bulid if the previous result showed
124 - CommandResult object containing the results of the build
125 - boolean indicating whether 'make config' is still needed
127 # Create a default result - it will be overwritte by the call to
128 # self.Make() below, in the event that we do a build.
129 result = command.CommandResult()
130 result.return_code = 0
131 if self.builder.in_tree:
134 out_dir = os.path.join(work_dir, 'build')
136 # Check if the job was already completed last time
137 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
138 result.already_done = os.path.exists(done_file)
139 will_build = (force_build or force_build_failures or
140 not result.already_done)
141 if result.already_done and will_build:
142 # Get the return code from that build and use it
143 with open(done_file, 'r') as fd:
144 result.return_code = int(fd.readline())
145 err_file = self.builder.GetErrFile(commit_upto, brd.target)
146 if os.path.exists(err_file) and os.stat(err_file).st_size:
147 result.stderr = 'bad'
148 elif not force_build:
149 # The build passed, so no need to build it again
153 # We are going to have to build it. First, get a toolchain
154 if not self.toolchain:
156 self.toolchain = self.builder.toolchains.Select(brd.arch)
157 except ValueError as err:
158 result.return_code = 10
160 result.stderr = str(err)
161 # TODO(sjg@chromium.org): This gets swallowed, but needs
165 # Checkout the right commit
166 if self.builder.commits:
167 commit = self.builder.commits[commit_upto]
168 if self.builder.checkout:
169 git_dir = os.path.join(work_dir, '.git')
170 gitutil.Checkout(commit.hash, git_dir, work_dir,
175 # Set up the environment and command line
176 env = self.toolchain.MakeEnvironment()
180 src_dir = os.path.realpath(work_dir)
181 if not self.builder.in_tree:
182 if commit_upto is None:
183 # In this case we are building in the original source
184 # directory (i.e. the current directory where buildman
185 # is invoked. The output directory is set to this
186 # thread's selected work directory.
188 # Symlinks can confuse U-Boot's Makefile since
189 # we may use '..' in our path, so remove them.
190 work_dir = os.path.realpath(work_dir)
191 args.append('O=%s/build' % work_dir)
193 src_dir = os.getcwd()
195 args.append('O=build')
197 if self.builder.num_jobs is not None:
198 args.extend(['-j', str(self.builder.num_jobs)])
199 config_args = ['%s_defconfig' % brd.target]
201 args.extend(self.builder.toolchains.GetMakeArguments(brd))
203 # If we need to reconfigure, do that now
205 result = self.Make(commit, brd, 'mrproper', cwd,
206 'mrproper', *args, env=env)
207 result = self.Make(commit, brd, 'config', cwd,
208 *(args + config_args), env=env)
209 config_out = result.combined
210 do_config = False # No need to configure next time
211 if result.return_code == 0:
212 result = self.Make(commit, brd, 'build', cwd, *args,
214 result.stderr = result.stderr.replace(src_dir + '/', '')
216 result.return_code = 1
217 result.stderr = 'No tool chain for %s\n' % brd.arch
218 result.already_done = False
220 result.toolchain = self.toolchain
222 result.commit_upto = commit_upto
223 result.out_dir = out_dir
224 return result, do_config
226 def _WriteResult(self, result, keep_outputs):
227 """Write a built result to the output directory.
230 result: CommandResult object containing result to write
231 keep_outputs: True to store the output binaries, False
235 if result.return_code < 0:
239 if result.stderr and 'No child processes' in result.stderr:
242 if result.already_done:
245 # Write the output and stderr
246 output_dir = self.builder._GetOutputDir(result.commit_upto)
248 build_dir = self.builder.GetBuildDir(result.commit_upto,
252 outfile = os.path.join(build_dir, 'log')
253 with open(outfile, 'w') as fd:
255 fd.write(result.stdout)
257 errfile = self.builder.GetErrFile(result.commit_upto,
260 with open(errfile, 'w') as fd:
261 fd.write(result.stderr)
262 elif os.path.exists(errfile):
266 # Write the build result and toolchain information.
267 done_file = self.builder.GetDoneFile(result.commit_upto,
269 with open(done_file, 'w') as fd:
270 fd.write('%s' % result.return_code)
271 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
272 print >>fd, 'gcc', result.toolchain.gcc
273 print >>fd, 'path', result.toolchain.path
274 print >>fd, 'cross', result.toolchain.cross
275 print >>fd, 'arch', result.toolchain.arch
276 fd.write('%s' % result.return_code)
278 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
279 print >>fd, 'gcc', result.toolchain.gcc
280 print >>fd, 'path', result.toolchain.path
282 # Write out the image and function size information and an objdump
283 env = result.toolchain.MakeEnvironment()
285 for fname in ['u-boot', 'spl/u-boot-spl']:
286 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
287 nm_result = command.RunPipe([cmd], capture=True,
288 capture_stderr=True, cwd=result.out_dir,
289 raise_on_error=False, env=env)
291 nm = self.builder.GetFuncSizesFile(result.commit_upto,
292 result.brd.target, fname)
293 with open(nm, 'w') as fd:
294 print >>fd, nm_result.stdout,
296 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
297 dump_result = command.RunPipe([cmd], capture=True,
298 capture_stderr=True, cwd=result.out_dir,
299 raise_on_error=False, env=env)
301 if dump_result.stdout:
302 objdump = self.builder.GetObjdumpFile(result.commit_upto,
303 result.brd.target, fname)
304 with open(objdump, 'w') as fd:
305 print >>fd, dump_result.stdout,
306 for line in dump_result.stdout.splitlines():
307 fields = line.split()
308 if len(fields) > 5 and fields[1] == '.rodata':
309 rodata_size = fields[2]
311 cmd = ['%ssize' % self.toolchain.cross, fname]
312 size_result = command.RunPipe([cmd], capture=True,
313 capture_stderr=True, cwd=result.out_dir,
314 raise_on_error=False, env=env)
315 if size_result.stdout:
316 lines.append(size_result.stdout.splitlines()[1] + ' ' +
319 # Write out the image sizes file. This is similar to the output
320 # of binutil's 'size' utility, but it omits the header line and
321 # adds an additional hex value at the end of each line for the
324 sizes = self.builder.GetSizesFile(result.commit_upto,
326 with open(sizes, 'w') as fd:
327 print >>fd, '\n'.join(lines)
329 # Now write the actual build output
331 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
332 'include/autoconf.mk', 'spl/u-boot-spl',
333 'spl/u-boot-spl.bin']
334 for pattern in patterns:
335 file_list = glob.glob(os.path.join(result.out_dir, pattern))
336 for fname in file_list:
337 shutil.copy(fname, build_dir)
340 def RunJob(self, job):
343 A job consists of a building a list of commits for a particular board.
349 work_dir = self.builder.GetThreadDir(self.thread_num)
350 self.toolchain = None
352 # Run 'make board_defconfig' on the first commit
356 for commit_upto in range(0, len(job.commits), job.step):
357 result, request_config = self.RunCommit(commit_upto, brd,
359 force_build or self.builder.force_build,
360 self.builder.force_build_failures)
361 failed = result.return_code or result.stderr
362 did_config = do_config
363 if failed and not do_config:
364 # If our incremental build failed, try building again
366 if self.builder.force_config_on_failure:
367 result, request_config = self.RunCommit(commit_upto,
368 brd, work_dir, True, True, False)
370 if not self.builder.force_reconfig:
371 do_config = request_config
373 # If we built that commit, then config is done. But if we got
374 # an warning, reconfig next time to force it to build the same
375 # files that created warnings this time. Otherwise an
376 # incremental build may not build the same file, and we will
377 # think that the warning has gone away.
378 # We could avoid this by using -Werror everywhere...
379 # For errors, the problem doesn't happen, since presumably
380 # the build stopped and didn't generate output, so will retry
381 # that file next time. So we could detect warnings and deal
382 # with them specially here. For now, we just reconfigure if
383 # anything goes work.
384 # Of course this is substantially slower if there are build
385 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
387 if (failed and not result.already_done and not did_config and
388 self.builder.force_config_on_failure):
389 # If this build failed, try the next one with a
391 # Sometimes if the board_config.h file changes it can mess
392 # with dependencies, and we get:
393 # make: *** No rule to make target `include/autoconf.mk',
394 # needed by `depend'.
399 if self.builder.force_config_on_failure:
402 result.commit_upto = commit_upto
403 if result.return_code < 0:
404 raise ValueError('Interrupt')
406 # We have the build results, so output the result
407 self._WriteResult(result, job.keep_outputs)
408 self.builder.out_queue.put(result)
410 # Just build the currently checked-out build
411 result, request_config = self.RunCommit(None, brd, work_dir, True,
412 True, self.builder.force_build_failures)
413 result.commit_upto = 0
414 self._WriteResult(result, job.keep_outputs)
415 self.builder.out_queue.put(result)
418 """Our thread's run function
420 This thread picks a job from the queue, runs it, and then goes to the
425 job = self.builder.queue.get()
426 if self.builder.active and alive:
430 if self.builder.active and alive:
432 except Exception as err:
436 self.builder.queue.task_done()