1 # Copyright (c) 2014 Google, Inc
3 # SPDX-License-Identifier: GPL-2.0+
15 def Mkdir(dirname, parents = False):
16 """Make a directory if it doesn't already exist.
19 dirname: Directory to create
26 except OSError as err:
27 if err.errno == errno.EEXIST:
33 """Holds information about a job to be performed by a thread
36 board: Board object to build
37 commits: List of commit options to build.
44 class ResultThread(threading.Thread):
45 """This thread processes results from builder threads.
47 It simply passes the results on to the builder. There is only one
48 result thread, and this helps to serialise the build output.
50 def __init__(self, builder):
51 """Set up a new result thread
54 builder: Builder which will be sent each result
56 threading.Thread.__init__(self)
57 self.builder = builder
60 """Called to start up the result thread.
62 We collect the next result job and pass it on to the build.
65 result = self.builder.out_queue.get()
66 self.builder.ProcessResult(result)
67 self.builder.out_queue.task_done()
70 class BuilderThread(threading.Thread):
71 """This thread builds U-Boot for a particular board.
73 An input queue provides each new job. We run 'make' to build U-Boot
74 and then pass the results on to the output queue.
77 builder: The builder which contains information we might need
78 thread_num: Our thread number (0-n-1), used to decide on a
81 def __init__(self, builder, thread_num):
82 """Set up a new builder thread"""
83 threading.Thread.__init__(self)
84 self.builder = builder
85 self.thread_num = thread_num
87 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
88 """Run 'make' on a particular commit and board.
90 The source code will already be checked out, so the 'commit'
91 argument is only for information.
94 commit: Commit object that is being built
95 brd: Board object that is being built
96 stage: Stage of the build. Valid stages are:
97 mrproper - can be called to clean source
98 config - called to configure for a board
99 build - the main make invocation - it does the build
100 args: A list of arguments to pass to 'make'
101 kwargs: A list of keyword arguments to pass to command.RunPipe()
106 return self.builder.do_make(commit, brd, stage, cwd, *args,
109 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
110 force_build_failures):
111 """Build a particular commit.
113 If the build is already done, and we are not forcing a build, we skip
114 the build and just return the previously-saved results.
117 commit_upto: Commit number to build (0...n-1)
118 brd: Board object to build
119 work_dir: Directory to which the source will be checked out
120 do_config: True to run a make <board>_defconfig on the source
121 force_build: Force a build even if one was previously done
122 force_build_failures: Force a bulid if the previous result showed
127 - CommandResult object containing the results of the build
128 - boolean indicating whether 'make config' is still needed
130 # Create a default result - it will be overwritte by the call to
131 # self.Make() below, in the event that we do a build.
132 result = command.CommandResult()
133 result.return_code = 0
134 if self.builder.in_tree:
137 out_dir = os.path.join(work_dir, 'build')
139 # Check if the job was already completed last time
140 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
141 result.already_done = os.path.exists(done_file)
142 will_build = (force_build or force_build_failures or
143 not result.already_done)
144 if result.already_done:
145 # Get the return code from that build and use it
146 with open(done_file, 'r') as fd:
147 result.return_code = int(fd.readline())
149 err_file = self.builder.GetErrFile(commit_upto, brd.target)
150 if os.path.exists(err_file) and os.stat(err_file).st_size:
151 result.stderr = 'bad'
152 elif not force_build:
153 # The build passed, so no need to build it again
157 # We are going to have to build it. First, get a toolchain
158 if not self.toolchain:
160 self.toolchain = self.builder.toolchains.Select(brd.arch)
161 except ValueError as err:
162 result.return_code = 10
164 result.stderr = str(err)
165 # TODO(sjg@chromium.org): This gets swallowed, but needs
169 # Checkout the right commit
170 if self.builder.commits:
171 commit = self.builder.commits[commit_upto]
172 if self.builder.checkout:
173 git_dir = os.path.join(work_dir, '.git')
174 gitutil.Checkout(commit.hash, git_dir, work_dir,
179 # Set up the environment and command line
180 env = self.toolchain.MakeEnvironment()
184 src_dir = os.path.realpath(work_dir)
185 if not self.builder.in_tree:
186 if commit_upto is None:
187 # In this case we are building in the original source
188 # directory (i.e. the current directory where buildman
189 # is invoked. The output directory is set to this
190 # thread's selected work directory.
192 # Symlinks can confuse U-Boot's Makefile since
193 # we may use '..' in our path, so remove them.
194 work_dir = os.path.realpath(work_dir)
195 args.append('O=%s/build' % work_dir)
197 src_dir = os.getcwd()
199 args.append('O=build')
201 if self.builder.num_jobs is not None:
202 args.extend(['-j', str(self.builder.num_jobs)])
203 config_args = ['%s_defconfig' % brd.target]
205 args.extend(self.builder.toolchains.GetMakeArguments(brd))
207 # If we need to reconfigure, do that now
209 result = self.Make(commit, brd, 'mrproper', cwd,
210 'mrproper', *args, env=env)
211 result = self.Make(commit, brd, 'config', cwd,
212 *(args + config_args), env=env)
213 config_out = result.combined
214 do_config = False # No need to configure next time
215 if result.return_code == 0:
216 result = self.Make(commit, brd, 'build', cwd, *args,
218 result.stderr = result.stderr.replace(src_dir + '/', '')
220 result.return_code = 1
221 result.stderr = 'No tool chain for %s\n' % brd.arch
222 result.already_done = False
224 result.toolchain = self.toolchain
226 result.commit_upto = commit_upto
227 result.out_dir = out_dir
228 return result, do_config
230 def _WriteResult(self, result, keep_outputs):
231 """Write a built result to the output directory.
234 result: CommandResult object containing result to write
235 keep_outputs: True to store the output binaries, False
239 if result.return_code < 0:
243 if result.stderr and 'No child processes' in result.stderr:
246 if result.already_done:
249 # Write the output and stderr
250 output_dir = self.builder._GetOutputDir(result.commit_upto)
252 build_dir = self.builder.GetBuildDir(result.commit_upto,
256 outfile = os.path.join(build_dir, 'log')
257 with open(outfile, 'w') as fd:
259 fd.write(result.stdout)
261 errfile = self.builder.GetErrFile(result.commit_upto,
264 with open(errfile, 'w') as fd:
265 fd.write(result.stderr)
266 elif os.path.exists(errfile):
270 # Write the build result and toolchain information.
271 done_file = self.builder.GetDoneFile(result.commit_upto,
273 with open(done_file, 'w') as fd:
274 fd.write('%s' % result.return_code)
275 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
276 print >>fd, 'gcc', result.toolchain.gcc
277 print >>fd, 'path', result.toolchain.path
278 print >>fd, 'cross', result.toolchain.cross
279 print >>fd, 'arch', result.toolchain.arch
280 fd.write('%s' % result.return_code)
282 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
283 print >>fd, 'gcc', result.toolchain.gcc
284 print >>fd, 'path', result.toolchain.path
286 # Write out the image and function size information and an objdump
287 env = result.toolchain.MakeEnvironment()
289 for fname in ['u-boot', 'spl/u-boot-spl']:
290 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
291 nm_result = command.RunPipe([cmd], capture=True,
292 capture_stderr=True, cwd=result.out_dir,
293 raise_on_error=False, env=env)
295 nm = self.builder.GetFuncSizesFile(result.commit_upto,
296 result.brd.target, fname)
297 with open(nm, 'w') as fd:
298 print >>fd, nm_result.stdout,
300 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
301 dump_result = command.RunPipe([cmd], capture=True,
302 capture_stderr=True, cwd=result.out_dir,
303 raise_on_error=False, env=env)
305 if dump_result.stdout:
306 objdump = self.builder.GetObjdumpFile(result.commit_upto,
307 result.brd.target, fname)
308 with open(objdump, 'w') as fd:
309 print >>fd, dump_result.stdout,
310 for line in dump_result.stdout.splitlines():
311 fields = line.split()
312 if len(fields) > 5 and fields[1] == '.rodata':
313 rodata_size = fields[2]
315 cmd = ['%ssize' % self.toolchain.cross, fname]
316 size_result = command.RunPipe([cmd], capture=True,
317 capture_stderr=True, cwd=result.out_dir,
318 raise_on_error=False, env=env)
319 if size_result.stdout:
320 lines.append(size_result.stdout.splitlines()[1] + ' ' +
323 # Write out the image sizes file. This is similar to the output
324 # of binutil's 'size' utility, but it omits the header line and
325 # adds an additional hex value at the end of each line for the
328 sizes = self.builder.GetSizesFile(result.commit_upto,
330 with open(sizes, 'w') as fd:
331 print >>fd, '\n'.join(lines)
333 # Now write the actual build output
335 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
336 'include/autoconf.mk', 'spl/u-boot-spl',
337 'spl/u-boot-spl.bin']
338 for pattern in patterns:
339 file_list = glob.glob(os.path.join(result.out_dir, pattern))
340 for fname in file_list:
341 shutil.copy(fname, build_dir)
344 def RunJob(self, job):
347 A job consists of a building a list of commits for a particular board.
353 work_dir = self.builder.GetThreadDir(self.thread_num)
354 self.toolchain = None
356 # Run 'make board_defconfig' on the first commit
360 for commit_upto in range(0, len(job.commits), job.step):
361 result, request_config = self.RunCommit(commit_upto, brd,
363 force_build or self.builder.force_build,
364 self.builder.force_build_failures)
365 failed = result.return_code or result.stderr
366 did_config = do_config
367 if failed and not do_config:
368 # If our incremental build failed, try building again
370 if self.builder.force_config_on_failure:
371 result, request_config = self.RunCommit(commit_upto,
372 brd, work_dir, True, True, False)
374 if not self.builder.force_reconfig:
375 do_config = request_config
377 # If we built that commit, then config is done. But if we got
378 # an warning, reconfig next time to force it to build the same
379 # files that created warnings this time. Otherwise an
380 # incremental build may not build the same file, and we will
381 # think that the warning has gone away.
382 # We could avoid this by using -Werror everywhere...
383 # For errors, the problem doesn't happen, since presumably
384 # the build stopped and didn't generate output, so will retry
385 # that file next time. So we could detect warnings and deal
386 # with them specially here. For now, we just reconfigure if
387 # anything goes work.
388 # Of course this is substantially slower if there are build
389 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
391 if (failed and not result.already_done and not did_config and
392 self.builder.force_config_on_failure):
393 # If this build failed, try the next one with a
395 # Sometimes if the board_config.h file changes it can mess
396 # with dependencies, and we get:
397 # make: *** No rule to make target `include/autoconf.mk',
398 # needed by `depend'.
403 if self.builder.force_config_on_failure:
406 result.commit_upto = commit_upto
407 if result.return_code < 0:
408 raise ValueError('Interrupt')
410 # We have the build results, so output the result
411 self._WriteResult(result, job.keep_outputs)
412 self.builder.out_queue.put(result)
414 # Just build the currently checked-out build
415 result, request_config = self.RunCommit(None, brd, work_dir, True,
416 True, self.builder.force_build_failures)
417 result.commit_upto = 0
418 self._WriteResult(result, job.keep_outputs)
419 self.builder.out_queue.put(result)
422 """Our thread's run function
424 This thread picks a job from the queue, runs it, and then goes to the
429 job = self.builder.queue.get()
430 if self.builder.active and alive:
434 if self.builder.active and alive:
436 except Exception as err:
440 self.builder.queue.task_done()