1 # Copyright (c) 2014 Google, Inc
3 # SPDX-License-Identifier: GPL-2.0+
16 """Make a directory if it doesn't already exist.
19 dirname: Directory to create
23 except OSError as err:
24 if err.errno == errno.EEXIST:
30 """Holds information about a job to be performed by a thread
33 board: Board object to build
34 commits: List of commit options to build.
41 class ResultThread(threading.Thread):
42 """This thread processes results from builder threads.
44 It simply passes the results on to the builder. There is only one
45 result thread, and this helps to serialise the build output.
47 def __init__(self, builder):
48 """Set up a new result thread
51 builder: Builder which will be sent each result
53 threading.Thread.__init__(self)
54 self.builder = builder
57 """Called to start up the result thread.
59 We collect the next result job and pass it on to the build.
62 result = self.builder.out_queue.get()
63 self.builder.ProcessResult(result)
64 self.builder.out_queue.task_done()
67 class BuilderThread(threading.Thread):
68 """This thread builds U-Boot for a particular board.
70 An input queue provides each new job. We run 'make' to build U-Boot
71 and then pass the results on to the output queue.
74 builder: The builder which contains information we might need
75 thread_num: Our thread number (0-n-1), used to decide on a
78 def __init__(self, builder, thread_num):
79 """Set up a new builder thread"""
80 threading.Thread.__init__(self)
81 self.builder = builder
82 self.thread_num = thread_num
84 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
85 """Run 'make' on a particular commit and board.
87 The source code will already be checked out, so the 'commit'
88 argument is only for information.
91 commit: Commit object that is being built
92 brd: Board object that is being built
93 stage: Stage of the build. Valid stages are:
94 mrproper - can be called to clean source
95 config - called to configure for a board
96 build - the main make invocation - it does the build
97 args: A list of arguments to pass to 'make'
98 kwargs: A list of keyword arguments to pass to command.RunPipe()
103 return self.builder.do_make(commit, brd, stage, cwd, *args,
106 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
107 force_build_failures):
108 """Build a particular commit.
110 If the build is already done, and we are not forcing a build, we skip
111 the build and just return the previously-saved results.
114 commit_upto: Commit number to build (0...n-1)
115 brd: Board object to build
116 work_dir: Directory to which the source will be checked out
117 do_config: True to run a make <board>_defconfig on the source
118 force_build: Force a build even if one was previously done
119 force_build_failures: Force a bulid if the previous result showed
124 - CommandResult object containing the results of the build
125 - boolean indicating whether 'make config' is still needed
127 # Create a default result - it will be overwritte by the call to
128 # self.Make() below, in the event that we do a build.
129 result = command.CommandResult()
130 result.return_code = 0
131 if self.builder.in_tree:
134 out_dir = os.path.join(work_dir, 'build')
136 # Check if the job was already completed last time
137 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
138 result.already_done = os.path.exists(done_file)
139 will_build = (force_build or force_build_failures or
140 not result.already_done)
141 if result.already_done and will_build:
142 # Get the return code from that build and use it
143 with open(done_file, 'r') as fd:
144 result.return_code = int(fd.readline())
145 err_file = self.builder.GetErrFile(commit_upto, brd.target)
146 if os.path.exists(err_file) and os.stat(err_file).st_size:
147 result.stderr = 'bad'
148 elif not force_build:
149 # The build passed, so no need to build it again
153 # We are going to have to build it. First, get a toolchain
154 if not self.toolchain:
156 self.toolchain = self.builder.toolchains.Select(brd.arch)
157 except ValueError as err:
158 result.return_code = 10
160 result.stderr = str(err)
161 # TODO(sjg@chromium.org): This gets swallowed, but needs
165 # Checkout the right commit
166 if self.builder.commits:
167 commit = self.builder.commits[commit_upto]
168 if self.builder.checkout:
169 git_dir = os.path.join(work_dir, '.git')
170 gitutil.Checkout(commit.hash, git_dir, work_dir,
175 # Set up the environment and command line
176 env = self.toolchain.MakeEnvironment()
180 if not self.builder.in_tree:
181 if commit_upto is None:
182 # In this case we are building in the original source
183 # directory (i.e. the current directory where buildman
184 # is invoked. The output directory is set to this
185 # thread's selected work directory.
187 # Symlinks can confuse U-Boot's Makefile since
188 # we may use '..' in our path, so remove them.
189 work_dir = os.path.realpath(work_dir)
190 args.append('O=%s/build' % work_dir)
193 args.append('O=build')
195 if self.builder.num_jobs is not None:
196 args.extend(['-j', str(self.builder.num_jobs)])
197 config_args = ['%s_defconfig' % brd.target]
199 args.extend(self.builder.toolchains.GetMakeArguments(brd))
201 # If we need to reconfigure, do that now
203 result = self.Make(commit, brd, 'mrproper', cwd,
204 'mrproper', *args, env=env)
205 result = self.Make(commit, brd, 'config', cwd,
206 *(args + config_args), env=env)
207 config_out = result.combined
208 do_config = False # No need to configure next time
209 if result.return_code == 0:
210 result = self.Make(commit, brd, 'build', cwd, *args,
212 result.stdout = config_out + result.stdout
214 result.return_code = 1
215 result.stderr = 'No tool chain for %s\n' % brd.arch
216 result.already_done = False
218 result.toolchain = self.toolchain
220 result.commit_upto = commit_upto
221 result.out_dir = out_dir
222 return result, do_config
224 def _WriteResult(self, result, keep_outputs):
225 """Write a built result to the output directory.
228 result: CommandResult object containing result to write
229 keep_outputs: True to store the output binaries, False
233 if result.return_code < 0:
237 if result.stderr and 'No child processes' in result.stderr:
240 if result.already_done:
243 # Write the output and stderr
244 output_dir = self.builder._GetOutputDir(result.commit_upto)
246 build_dir = self.builder.GetBuildDir(result.commit_upto,
250 outfile = os.path.join(build_dir, 'log')
251 with open(outfile, 'w') as fd:
253 fd.write(result.stdout)
255 errfile = self.builder.GetErrFile(result.commit_upto,
258 with open(errfile, 'w') as fd:
259 fd.write(result.stderr)
260 elif os.path.exists(errfile):
264 # Write the build result and toolchain information.
265 done_file = self.builder.GetDoneFile(result.commit_upto,
267 with open(done_file, 'w') as fd:
268 fd.write('%s' % result.return_code)
269 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
270 print >>fd, 'gcc', result.toolchain.gcc
271 print >>fd, 'path', result.toolchain.path
272 print >>fd, 'cross', result.toolchain.cross
273 print >>fd, 'arch', result.toolchain.arch
274 fd.write('%s' % result.return_code)
276 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
277 print >>fd, 'gcc', result.toolchain.gcc
278 print >>fd, 'path', result.toolchain.path
280 # Write out the image and function size information and an objdump
281 env = result.toolchain.MakeEnvironment()
283 for fname in ['u-boot', 'spl/u-boot-spl']:
284 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
285 nm_result = command.RunPipe([cmd], capture=True,
286 capture_stderr=True, cwd=result.out_dir,
287 raise_on_error=False, env=env)
289 nm = self.builder.GetFuncSizesFile(result.commit_upto,
290 result.brd.target, fname)
291 with open(nm, 'w') as fd:
292 print >>fd, nm_result.stdout,
294 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
295 dump_result = command.RunPipe([cmd], capture=True,
296 capture_stderr=True, cwd=result.out_dir,
297 raise_on_error=False, env=env)
299 if dump_result.stdout:
300 objdump = self.builder.GetObjdumpFile(result.commit_upto,
301 result.brd.target, fname)
302 with open(objdump, 'w') as fd:
303 print >>fd, dump_result.stdout,
304 for line in dump_result.stdout.splitlines():
305 fields = line.split()
306 if len(fields) > 5 and fields[1] == '.rodata':
307 rodata_size = fields[2]
309 cmd = ['%ssize' % self.toolchain.cross, fname]
310 size_result = command.RunPipe([cmd], capture=True,
311 capture_stderr=True, cwd=result.out_dir,
312 raise_on_error=False, env=env)
313 if size_result.stdout:
314 lines.append(size_result.stdout.splitlines()[1] + ' ' +
317 # Write out the image sizes file. This is similar to the output
318 # of binutil's 'size' utility, but it omits the header line and
319 # adds an additional hex value at the end of each line for the
322 sizes = self.builder.GetSizesFile(result.commit_upto,
324 with open(sizes, 'w') as fd:
325 print >>fd, '\n'.join(lines)
327 # Now write the actual build output
329 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
330 'include/autoconf.mk', 'spl/u-boot-spl',
331 'spl/u-boot-spl.bin']
332 for pattern in patterns:
333 file_list = glob.glob(os.path.join(result.out_dir, pattern))
334 for fname in file_list:
335 shutil.copy(fname, build_dir)
338 def RunJob(self, job):
341 A job consists of a building a list of commits for a particular board.
347 work_dir = self.builder.GetThreadDir(self.thread_num)
348 self.toolchain = None
350 # Run 'make board_defconfig' on the first commit
354 for commit_upto in range(0, len(job.commits), job.step):
355 result, request_config = self.RunCommit(commit_upto, brd,
357 force_build or self.builder.force_build,
358 self.builder.force_build_failures)
359 failed = result.return_code or result.stderr
360 did_config = do_config
361 if failed and not do_config:
362 # If our incremental build failed, try building again
364 if self.builder.force_config_on_failure:
365 result, request_config = self.RunCommit(commit_upto,
366 brd, work_dir, True, True, False)
368 if not self.builder.force_reconfig:
369 do_config = request_config
371 # If we built that commit, then config is done. But if we got
372 # an warning, reconfig next time to force it to build the same
373 # files that created warnings this time. Otherwise an
374 # incremental build may not build the same file, and we will
375 # think that the warning has gone away.
376 # We could avoid this by using -Werror everywhere...
377 # For errors, the problem doesn't happen, since presumably
378 # the build stopped and didn't generate output, so will retry
379 # that file next time. So we could detect warnings and deal
380 # with them specially here. For now, we just reconfigure if
381 # anything goes work.
382 # Of course this is substantially slower if there are build
383 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
385 if (failed and not result.already_done and not did_config and
386 self.builder.force_config_on_failure):
387 # If this build failed, try the next one with a
389 # Sometimes if the board_config.h file changes it can mess
390 # with dependencies, and we get:
391 # make: *** No rule to make target `include/autoconf.mk',
392 # needed by `depend'.
397 if self.builder.force_config_on_failure:
400 result.commit_upto = commit_upto
401 if result.return_code < 0:
402 raise ValueError('Interrupt')
404 # We have the build results, so output the result
405 self._WriteResult(result, job.keep_outputs)
406 self.builder.out_queue.put(result)
408 # Just build the currently checked-out build
409 result, request_config = self.RunCommit(None, brd, work_dir, True,
410 True, self.builder.force_build_failures)
411 result.commit_upto = 0
412 self._WriteResult(result, job.keep_outputs)
413 self.builder.out_queue.put(result)
416 """Our thread's run function
418 This thread picks a job from the queue, runs it, and then goes to the
423 job = self.builder.queue.get()
424 if self.builder.active and alive:
428 if self.builder.active and alive:
430 except Exception as err:
434 self.builder.queue.task_done()