1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2012 The Chromium OS Authors.
7 from HTMLParser import HTMLParser
17 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
18 PRIORITY_CALC) = range(4)
20 # Simple class to collect links from a page
21 class MyHTMLParser(HTMLParser):
22 def __init__(self, arch):
23 """Create a new parser
25 After the parser runs, self.links will be set to a list of the links
26 to .xz archives found in the page, and self.arch_link will be set to
27 the one for the given architecture (or None if not found).
30 arch: Architecture to search for
32 HTMLParser.__init__(self)
35 self.re_arch = re.compile('[-_]%s-' % arch)
37 def handle_starttag(self, tag, attrs):
39 for tag, value in attrs:
41 if value and value.endswith('.xz'):
42 self.links.append(value)
43 if self.re_arch.search(value):
44 self.arch_link = value
51 gcc: Full path to C compiler
52 path: Directory path containing C compiler
53 cross: Cross compile string, e.g. 'arm-linux-'
54 arch: Architecture of toolchain as determined from the first
55 component of the filename. E.g. arm-linux-gcc becomes arm
56 priority: Toolchain priority (0=highest, 20=lowest)
57 override_toolchain: Toolchain to use for sandbox, overriding the normal
60 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
61 arch=None, override_toolchain=None):
62 """Create a new toolchain object.
65 fname: Filename of the gcc component
66 test: True to run the toolchain to test it
67 verbose: True to print out the information
68 priority: Priority to use for this toolchain, or PRIORITY_CALC to
72 self.path = os.path.dirname(fname)
73 self.override_toolchain = override_toolchain
75 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
76 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
77 basename = os.path.basename(fname)
78 pos = basename.rfind('-')
79 self.cross = basename[:pos + 1] if pos != -1 else ''
81 # The architecture is the first part of the name
82 pos = self.cross.find('-')
86 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
87 if self.arch == 'sandbox' and override_toolchain:
88 self.gcc = override_toolchain
90 env = self.MakeEnvironment(False)
92 # As a basic sanity check, run the C compiler with --version
93 cmd = [fname, '--version']
94 if priority == PRIORITY_CALC:
95 self.priority = self.GetPriority(fname)
97 self.priority = priority
99 result = command.RunPipe([cmd], capture=True, env=env,
100 raise_on_error=False)
101 self.ok = result.return_code == 0
103 print 'Tool chain test: ',
105 print "OK, arch='%s', priority %d" % (self.arch,
109 print 'Command: ', cmd
115 def GetPriority(self, fname):
116 """Return the priority of the toolchain.
118 Toolchains are ranked according to their suitability by their
122 fname: Filename of toolchain
124 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
126 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
127 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
128 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
129 '-linux-gnueabihf', '-le-linux', '-uclinux']
130 for prio in range(len(priority_list)):
131 if priority_list[prio] in fname:
132 return PRIORITY_CALC + prio
133 return PRIORITY_CALC + prio
135 def GetWrapper(self, show_warning=True):
136 """Get toolchain wrapper from the setting file.
139 for name, value in bsettings.GetItems('toolchain-wrapper'):
141 print "Warning: Wrapper not found"
147 def MakeEnvironment(self, full_path):
148 """Returns an environment for using the toolchain.
150 Thie takes the current environment and adds CROSS_COMPILE so that
151 the tool chain will operate correctly. This also disables localized
152 output and possibly unicode encoded output of all build tools by
156 full_path: Return the full path in CROSS_COMPILE and don't set
159 Dict containing the environemnt to use. This is based on the current
160 environment, with changes as needed to CROSS_COMPILE, PATH and
163 env = dict(os.environ)
164 wrapper = self.GetWrapper()
166 if self.override_toolchain:
167 # We'll use MakeArgs() to provide this
170 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
172 env['CROSS_COMPILE'] = wrapper + self.cross
173 env['PATH'] = self.path + ':' + env['PATH']
180 """Create the 'make' arguments for a toolchain
182 This is only used when the toolchain is being overridden. Since the
183 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
184 environment (and MakeEnvironment()) to override these values. This
185 function returns the arguments to accomplish this.
188 List of arguments to pass to 'make'
190 if self.override_toolchain:
191 return ['HOSTCC=%s' % self.override_toolchain,
192 'CC=%s' % self.override_toolchain]
197 """Manage a list of toolchains for building U-Boot
199 We select one toolchain for each architecture type
202 toolchains: Dict of Toolchain objects, keyed by architecture name
203 prefixes: Dict of prefixes to check, keyed by architecture. This can
204 be a full path and toolchain prefix, for example
205 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
206 something on the search path, for example
207 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
208 paths: List of paths to check for toolchains (may contain wildcards)
211 def __init__(self, override_toolchain=None):
215 self.override_toolchain = override_toolchain
216 self._make_flags = dict(bsettings.GetItems('make-flags'))
218 def GetPathList(self, show_warning=True):
219 """Get a list of available toolchain paths
222 show_warning: True to show a warning if there are no tool chains.
225 List of strings, each a path to a toolchain mentioned in the
226 [toolchain] section of the settings file.
228 toolchains = bsettings.GetItems('toolchain')
229 if show_warning and not toolchains:
230 print ("Warning: No tool chains. Please run 'buildman "
231 "--fetch-arch all' to download all available toolchains, or "
232 "add a [toolchain] section to your buildman config file "
233 "%s. See README for details" %
234 bsettings.config_fname)
237 for name, value in toolchains:
239 paths += glob.glob(value)
244 def GetSettings(self, show_warning=True):
245 """Get toolchain settings from the settings file.
248 show_warning: True to show a warning if there are no tool chains.
250 self.prefixes = bsettings.GetItems('toolchain-prefix')
251 self.paths += self.GetPathList(show_warning)
253 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
255 """Add a toolchain to our list
257 We select the given toolchain as our preferred one for its
258 architecture if it is a higher priority than the others.
261 fname: Filename of toolchain's gcc driver
262 test: True to run the toolchain to test it
263 priority: Priority to use for this toolchain
264 arch: Toolchain architecture, or None if not known
266 toolchain = Toolchain(fname, test, verbose, priority, arch,
267 self.override_toolchain)
268 add_it = toolchain.ok
269 if toolchain.arch in self.toolchains:
270 add_it = (toolchain.priority <
271 self.toolchains[toolchain.arch].priority)
273 self.toolchains[toolchain.arch] = toolchain
275 print ("Toolchain '%s' at priority %d will be ignored because "
276 "another toolchain for arch '%s' has priority %d" %
277 (toolchain.gcc, toolchain.priority, toolchain.arch,
278 self.toolchains[toolchain.arch].priority))
280 def ScanPath(self, path, verbose):
281 """Scan a path for a valid toolchain
285 verbose: True to print out progress information
287 Filename of C compiler if found, else None
290 for subdir in ['.', 'bin', 'usr/bin']:
291 dirname = os.path.join(path, subdir)
292 if verbose: print " - looking in '%s'" % dirname
293 for fname in glob.glob(dirname + '/*gcc'):
294 if verbose: print " - found '%s'" % fname
298 def ScanPathEnv(self, fname):
299 """Scan the PATH environment variable for a given filename.
302 fname: Filename to scan for
304 List of matching pathanames, or [] if none
307 for path in os.environ["PATH"].split(os.pathsep):
308 path = path.strip('"')
309 pathname = os.path.join(path, fname)
310 if os.path.exists(pathname):
311 pathname_list.append(pathname)
314 def Scan(self, verbose):
315 """Scan for available toolchains and select the best for each arch.
317 We look for all the toolchains we can file, figure out the
318 architecture for each, and whether it works. Then we select the
319 highest priority toolchain for each arch.
322 verbose: True to print out progress information
324 if verbose: print 'Scanning for tool chains'
325 for name, value in self.prefixes:
326 if verbose: print " - scanning prefix '%s'" % value
327 if os.path.exists(value):
328 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
330 fname = value + 'gcc'
331 if os.path.exists(fname):
332 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
334 fname_list = self.ScanPathEnv(fname)
336 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
338 raise ValueError, ("No tool chain found for prefix '%s'" %
340 for path in self.paths:
341 if verbose: print " - scanning path '%s'" % path
342 fnames = self.ScanPath(path, verbose)
344 self.Add(fname, True, verbose)
347 """List out the selected toolchains for each architecture"""
348 col = terminal.Color()
349 print col.Color(col.BLUE, 'List of available toolchains (%d):' %
350 len(self.toolchains))
351 if len(self.toolchains):
352 for key, value in sorted(self.toolchains.iteritems()):
353 print '%-10s: %s' % (key, value.gcc)
357 def Select(self, arch):
358 """Returns the toolchain for a given architecture
361 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
364 toolchain object, or None if none found
366 for tag, value in bsettings.GetItems('toolchain-alias'):
368 for alias in value.split():
369 if alias in self.toolchains:
370 return self.toolchains[alias]
372 if not arch in self.toolchains:
373 raise ValueError, ("No tool chain found for arch '%s'" % arch)
374 return self.toolchains[arch]
376 def ResolveReferences(self, var_dict, args):
377 """Resolve variable references in a string
379 This converts ${blah} within the string to the value of blah.
380 This function works recursively.
383 var_dict: Dictionary containing variables and their values
384 args: String containing make arguments
388 >>> bsettings.Setup()
389 >>> tcs = Toolchains()
390 >>> tcs.Add('fred', False)
391 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
393 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
395 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
396 'this=OBLIQUE_setfi2ndrstnd'
398 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
401 m = re_var.search(args)
404 lookup = m.group(0)[2:-1]
405 value = var_dict.get(lookup, '')
406 args = args[:m.start(0)] + value + args[m.end(0):]
409 def GetMakeArguments(self, board):
410 """Returns 'make' arguments for a given board
412 The flags are in a section called 'make-flags'. Flags are named
413 after the target they represent, for example snapper9260=TESTING=1
414 will pass TESTING=1 to make when building the snapper9260 board.
416 References to other boards can be added in the string also. For
420 at91-boards=ENABLE_AT91_TEST=1
421 snapper9260=${at91-boards} BUILD_TAG=442
422 snapper9g45=${at91-boards} BUILD_TAG=443
424 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
425 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
427 A special 'target' variable is set to the board target.
430 board: Board object for the board to check.
432 'make' flags for that board, or '' if none
434 self._make_flags['target'] = board.target
435 arg_str = self.ResolveReferences(self._make_flags,
436 self._make_flags.get(board.target, ''))
437 args = arg_str.split(' ')
446 def LocateArchUrl(self, fetch_arch):
447 """Find a toolchain available online
449 Look in standard places for available toolchains. At present the
450 only standard place is at kernel.org.
453 arch: Architecture to look for, or 'list' for all
455 If fetch_arch is 'list', a tuple:
456 Machine architecture (e.g. x86_64)
459 URL containing this toolchain, if avaialble, else None
461 arch = command.OutputOneLine('uname', '-m')
462 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
463 versions = ['7.3.0', '6.4.0', '4.9.4']
465 for version in versions:
466 url = '%s/%s/%s/' % (base, arch, version)
467 print 'Checking: %s' % url
468 response = urllib2.urlopen(url)
469 html = response.read()
470 parser = MyHTMLParser(fetch_arch)
472 if fetch_arch == 'list':
473 links += parser.links
474 elif parser.arch_link:
475 return url + parser.arch_link
476 if fetch_arch == 'list':
480 def Download(self, url):
481 """Download a file to a temporary directory
487 Temporary directory name
488 Full path to the downloaded archive file in that directory,
489 or None if there was an error while downloading
491 print 'Downloading: %s' % url
492 leaf = url.split('/')[-1]
493 tmpdir = tempfile.mkdtemp('.buildman')
494 response = urllib2.urlopen(url)
495 fname = os.path.join(tmpdir, leaf)
496 fd = open(fname, 'wb')
497 meta = response.info()
498 size = int(meta.getheaders('Content-Length')[0])
503 # Read the file in chunks and show progress as we go
505 buffer = response.read(block_size)
507 print chr(8) * (len(status) + 1), '\r',
512 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
514 status = status + chr(8) * (len(status) + 1)
519 print 'Error, failed to download'
524 def Unpack(self, fname, dest):
528 fname: Filename to unpack
529 dest: Destination directory
531 Directory name of the first entry in the archive, without the
534 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
535 dirs = stdout.splitlines()[1].split('/')[:2]
536 return '/'.join(dirs)
538 def TestSettingsHasPath(self, path):
539 """Check if buildman will find this toolchain
542 True if the path is in settings, False if not
544 paths = self.GetPathList(False)
548 """List architectures with available toolchains to download"""
549 host_arch, archives = self.LocateArchUrl('list')
550 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
552 for archive in archives:
553 # Remove the host architecture from the start
554 arch = re_arch.match(archive[len(host_arch):])
556 if arch.group(1) != '2.0' and arch.group(1) != '64':
557 arch_set.add(arch.group(1))
558 return sorted(arch_set)
560 def FetchAndInstall(self, arch):
561 """Fetch and install a new toolchain
564 Architecture to fetch, or 'list' to list
566 # Fist get the URL for this architecture
567 col = terminal.Color()
568 print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
569 url = self.LocateArchUrl(arch)
571 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
574 home = os.environ['HOME']
575 dest = os.path.join(home, '.buildman-toolchains')
576 if not os.path.exists(dest):
579 # Download the tar file for this toolchain and unpack it
580 tmpdir, tarfile = self.Download(url)
583 print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
585 path = self.Unpack(tarfile, dest)
590 # Check that the toolchain works
591 print col.Color(col.GREEN, 'Testing')
592 dirpath = os.path.join(dest, path)
593 compiler_fname_list = self.ScanPath(dirpath, True)
594 if not compiler_fname_list:
595 print 'Could not locate C compiler - fetch failed.'
597 if len(compiler_fname_list) != 1:
598 print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
599 ', '.join(compiler_fname_list))
600 toolchain = Toolchain(compiler_fname_list[0], True, True)
602 # Make sure that it will be found by buildman
603 if not self.TestSettingsHasPath(dirpath):
604 print ("Adding 'download' to config file '%s'" %
605 bsettings.config_fname)
606 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)