1 # Copyright (c) 2014 Google, Inc
3 # SPDX-License-Identifier: GPL-2.0+
15 def Mkdir(dirname, parents = False):
16 """Make a directory if it doesn't already exist.
19 dirname: Directory to create
26 except OSError as err:
27 if err.errno == errno.EEXIST:
33 """Holds information about a job to be performed by a thread
36 board: Board object to build
37 commits: List of commit options to build.
44 class ResultThread(threading.Thread):
45 """This thread processes results from builder threads.
47 It simply passes the results on to the builder. There is only one
48 result thread, and this helps to serialise the build output.
50 def __init__(self, builder):
51 """Set up a new result thread
54 builder: Builder which will be sent each result
56 threading.Thread.__init__(self)
57 self.builder = builder
60 """Called to start up the result thread.
62 We collect the next result job and pass it on to the build.
65 result = self.builder.out_queue.get()
66 self.builder.ProcessResult(result)
67 self.builder.out_queue.task_done()
70 class BuilderThread(threading.Thread):
71 """This thread builds U-Boot for a particular board.
73 An input queue provides each new job. We run 'make' to build U-Boot
74 and then pass the results on to the output queue.
77 builder: The builder which contains information we might need
78 thread_num: Our thread number (0-n-1), used to decide on a
81 def __init__(self, builder, thread_num):
82 """Set up a new builder thread"""
83 threading.Thread.__init__(self)
84 self.builder = builder
85 self.thread_num = thread_num
87 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
88 """Run 'make' on a particular commit and board.
90 The source code will already be checked out, so the 'commit'
91 argument is only for information.
94 commit: Commit object that is being built
95 brd: Board object that is being built
96 stage: Stage of the build. Valid stages are:
97 mrproper - can be called to clean source
98 config - called to configure for a board
99 build - the main make invocation - it does the build
100 args: A list of arguments to pass to 'make'
101 kwargs: A list of keyword arguments to pass to command.RunPipe()
106 return self.builder.do_make(commit, brd, stage, cwd, *args,
109 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
110 force_build_failures):
111 """Build a particular commit.
113 If the build is already done, and we are not forcing a build, we skip
114 the build and just return the previously-saved results.
117 commit_upto: Commit number to build (0...n-1)
118 brd: Board object to build
119 work_dir: Directory to which the source will be checked out
120 do_config: True to run a make <board>_defconfig on the source
121 force_build: Force a build even if one was previously done
122 force_build_failures: Force a bulid if the previous result showed
127 - CommandResult object containing the results of the build
128 - boolean indicating whether 'make config' is still needed
130 # Create a default result - it will be overwritte by the call to
131 # self.Make() below, in the event that we do a build.
132 result = command.CommandResult()
133 result.return_code = 0
134 if self.builder.in_tree:
137 out_dir = os.path.join(work_dir, 'build')
139 # Check if the job was already completed last time
140 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
141 result.already_done = os.path.exists(done_file)
142 will_build = (force_build or force_build_failures or
143 not result.already_done)
144 if result.already_done:
145 # Get the return code from that build and use it
146 with open(done_file, 'r') as fd:
147 result.return_code = int(fd.readline())
149 err_file = self.builder.GetErrFile(commit_upto, brd.target)
150 if os.path.exists(err_file) and os.stat(err_file).st_size:
151 result.stderr = 'bad'
152 elif not force_build:
153 # The build passed, so no need to build it again
157 # We are going to have to build it. First, get a toolchain
158 if not self.toolchain:
160 self.toolchain = self.builder.toolchains.Select(brd.arch)
161 except ValueError as err:
162 result.return_code = 10
164 result.stderr = str(err)
165 # TODO(sjg@chromium.org): This gets swallowed, but needs
169 # Checkout the right commit
170 if self.builder.commits:
171 commit = self.builder.commits[commit_upto]
172 if self.builder.checkout:
173 git_dir = os.path.join(work_dir, '.git')
174 gitutil.Checkout(commit.hash, git_dir, work_dir,
179 # Set up the environment and command line
180 env = self.toolchain.MakeEnvironment(self.builder.full_path)
184 src_dir = os.path.realpath(work_dir)
185 if not self.builder.in_tree:
186 if commit_upto is None:
187 # In this case we are building in the original source
188 # directory (i.e. the current directory where buildman
189 # is invoked. The output directory is set to this
190 # thread's selected work directory.
192 # Symlinks can confuse U-Boot's Makefile since
193 # we may use '..' in our path, so remove them.
194 work_dir = os.path.realpath(work_dir)
195 args.append('O=%s/build' % work_dir)
197 src_dir = os.getcwd()
199 args.append('O=build')
200 if not self.builder.verbose_build:
202 if self.builder.num_jobs is not None:
203 args.extend(['-j', str(self.builder.num_jobs)])
204 config_args = ['%s_defconfig' % brd.target]
206 args.extend(self.builder.toolchains.GetMakeArguments(brd))
208 # If we need to reconfigure, do that now
210 result = self.Make(commit, brd, 'mrproper', cwd,
211 'mrproper', *args, env=env)
212 result = self.Make(commit, brd, 'config', cwd,
213 *(args + config_args), env=env)
214 config_out = result.combined
215 do_config = False # No need to configure next time
216 if result.return_code == 0:
217 result = self.Make(commit, brd, 'build', cwd, *args,
219 result.stderr = result.stderr.replace(src_dir + '/', '')
221 result.return_code = 1
222 result.stderr = 'No tool chain for %s\n' % brd.arch
223 result.already_done = False
225 result.toolchain = self.toolchain
227 result.commit_upto = commit_upto
228 result.out_dir = out_dir
229 return result, do_config
231 def _WriteResult(self, result, keep_outputs):
232 """Write a built result to the output directory.
235 result: CommandResult object containing result to write
236 keep_outputs: True to store the output binaries, False
240 if result.return_code < 0:
244 if result.stderr and 'No child processes' in result.stderr:
247 if result.already_done:
250 # Write the output and stderr
251 output_dir = self.builder._GetOutputDir(result.commit_upto)
253 build_dir = self.builder.GetBuildDir(result.commit_upto,
257 outfile = os.path.join(build_dir, 'log')
258 with open(outfile, 'w') as fd:
260 fd.write(result.stdout)
262 errfile = self.builder.GetErrFile(result.commit_upto,
265 with open(errfile, 'w') as fd:
266 fd.write(result.stderr)
267 elif os.path.exists(errfile):
271 # Write the build result and toolchain information.
272 done_file = self.builder.GetDoneFile(result.commit_upto,
274 with open(done_file, 'w') as fd:
275 fd.write('%s' % result.return_code)
276 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
277 print >>fd, 'gcc', result.toolchain.gcc
278 print >>fd, 'path', result.toolchain.path
279 print >>fd, 'cross', result.toolchain.cross
280 print >>fd, 'arch', result.toolchain.arch
281 fd.write('%s' % result.return_code)
283 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
284 print >>fd, 'gcc', result.toolchain.gcc
285 print >>fd, 'path', result.toolchain.path
287 # Write out the image and function size information and an objdump
288 env = result.toolchain.MakeEnvironment(self.builder.full_path)
290 for fname in ['u-boot', 'spl/u-boot-spl']:
291 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
292 nm_result = command.RunPipe([cmd], capture=True,
293 capture_stderr=True, cwd=result.out_dir,
294 raise_on_error=False, env=env)
296 nm = self.builder.GetFuncSizesFile(result.commit_upto,
297 result.brd.target, fname)
298 with open(nm, 'w') as fd:
299 print >>fd, nm_result.stdout,
301 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
302 dump_result = command.RunPipe([cmd], capture=True,
303 capture_stderr=True, cwd=result.out_dir,
304 raise_on_error=False, env=env)
306 if dump_result.stdout:
307 objdump = self.builder.GetObjdumpFile(result.commit_upto,
308 result.brd.target, fname)
309 with open(objdump, 'w') as fd:
310 print >>fd, dump_result.stdout,
311 for line in dump_result.stdout.splitlines():
312 fields = line.split()
313 if len(fields) > 5 and fields[1] == '.rodata':
314 rodata_size = fields[2]
316 cmd = ['%ssize' % self.toolchain.cross, fname]
317 size_result = command.RunPipe([cmd], capture=True,
318 capture_stderr=True, cwd=result.out_dir,
319 raise_on_error=False, env=env)
320 if size_result.stdout:
321 lines.append(size_result.stdout.splitlines()[1] + ' ' +
324 # Write out the image sizes file. This is similar to the output
325 # of binutil's 'size' utility, but it omits the header line and
326 # adds an additional hex value at the end of each line for the
329 sizes = self.builder.GetSizesFile(result.commit_upto,
331 with open(sizes, 'w') as fd:
332 print >>fd, '\n'.join(lines)
334 # Now write the actual build output
336 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', '*.img',
337 'include/autoconf.mk', 'spl/u-boot-spl',
338 'spl/u-boot-spl.bin']
339 for pattern in patterns:
340 file_list = glob.glob(os.path.join(result.out_dir, pattern))
341 for fname in file_list:
342 shutil.copy(fname, build_dir)
345 def RunJob(self, job):
348 A job consists of a building a list of commits for a particular board.
354 work_dir = self.builder.GetThreadDir(self.thread_num)
355 self.toolchain = None
357 # Run 'make board_defconfig' on the first commit
361 for commit_upto in range(0, len(job.commits), job.step):
362 result, request_config = self.RunCommit(commit_upto, brd,
364 force_build or self.builder.force_build,
365 self.builder.force_build_failures)
366 failed = result.return_code or result.stderr
367 did_config = do_config
368 if failed and not do_config:
369 # If our incremental build failed, try building again
371 if self.builder.force_config_on_failure:
372 result, request_config = self.RunCommit(commit_upto,
373 brd, work_dir, True, True, False)
375 if not self.builder.force_reconfig:
376 do_config = request_config
378 # If we built that commit, then config is done. But if we got
379 # an warning, reconfig next time to force it to build the same
380 # files that created warnings this time. Otherwise an
381 # incremental build may not build the same file, and we will
382 # think that the warning has gone away.
383 # We could avoid this by using -Werror everywhere...
384 # For errors, the problem doesn't happen, since presumably
385 # the build stopped and didn't generate output, so will retry
386 # that file next time. So we could detect warnings and deal
387 # with them specially here. For now, we just reconfigure if
388 # anything goes work.
389 # Of course this is substantially slower if there are build
390 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
392 if (failed and not result.already_done and not did_config and
393 self.builder.force_config_on_failure):
394 # If this build failed, try the next one with a
396 # Sometimes if the board_config.h file changes it can mess
397 # with dependencies, and we get:
398 # make: *** No rule to make target `include/autoconf.mk',
399 # needed by `depend'.
404 if self.builder.force_config_on_failure:
407 result.commit_upto = commit_upto
408 if result.return_code < 0:
409 raise ValueError('Interrupt')
411 # We have the build results, so output the result
412 self._WriteResult(result, job.keep_outputs)
413 self.builder.out_queue.put(result)
415 # Just build the currently checked-out build
416 result, request_config = self.RunCommit(None, brd, work_dir, True,
417 True, self.builder.force_build_failures)
418 result.commit_upto = 0
419 self._WriteResult(result, job.keep_outputs)
420 self.builder.out_queue.put(result)
423 """Our thread's run function
425 This thread picks a job from the queue, runs it, and then goes to the
430 job = self.builder.queue.get()
431 if self.builder.active and alive:
435 if self.builder.active and alive:
437 except Exception as err:
441 self.builder.queue.task_done()