1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2012 The Chromium OS Authors.
7 from html.parser import HTMLParser
11 import urllib.request, urllib.error, urllib.parse
18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = list(range(4))
21 # Simple class to collect links from a page
22 class MyHTMLParser(HTMLParser):
23 def __init__(self, arch):
24 """Create a new parser
26 After the parser runs, self.links will be set to a list of the links
27 to .xz archives found in the page, and self.arch_link will be set to
28 the one for the given architecture (or None if not found).
31 arch: Architecture to search for
33 HTMLParser.__init__(self)
36 self.re_arch = re.compile('[-_]%s-' % arch)
38 def handle_starttag(self, tag, attrs):
40 for tag, value in attrs:
42 if value and value.endswith('.xz'):
43 self.links.append(value)
44 if self.re_arch.search(value):
45 self.arch_link = value
52 gcc: Full path to C compiler
53 path: Directory path containing C compiler
54 cross: Cross compile string, e.g. 'arm-linux-'
55 arch: Architecture of toolchain as determined from the first
56 component of the filename. E.g. arm-linux-gcc becomes arm
57 priority: Toolchain priority (0=highest, 20=lowest)
58 override_toolchain: Toolchain to use for sandbox, overriding the normal
61 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
62 arch=None, override_toolchain=None):
63 """Create a new toolchain object.
66 fname: Filename of the gcc component
67 test: True to run the toolchain to test it
68 verbose: True to print out the information
69 priority: Priority to use for this toolchain, or PRIORITY_CALC to
73 self.path = os.path.dirname(fname)
74 self.override_toolchain = override_toolchain
76 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
77 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
78 basename = os.path.basename(fname)
79 pos = basename.rfind('-')
80 self.cross = basename[:pos + 1] if pos != -1 else ''
82 # The architecture is the first part of the name
83 pos = self.cross.find('-')
87 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
88 if self.arch == 'sandbox' and override_toolchain:
89 self.gcc = override_toolchain
91 env = self.MakeEnvironment(False)
93 # As a basic sanity check, run the C compiler with --version
94 cmd = [fname, '--version']
95 if priority == PRIORITY_CALC:
96 self.priority = self.GetPriority(fname)
98 self.priority = priority
100 result = command.RunPipe([cmd], capture=True, env=env,
101 raise_on_error=False)
102 self.ok = result.return_code == 0
104 print('Tool chain test: ', end=' ')
106 print("OK, arch='%s', priority %d" % (self.arch,
110 print('Command: ', cmd)
116 def GetPriority(self, fname):
117 """Return the priority of the toolchain.
119 Toolchains are ranked according to their suitability by their
123 fname: Filename of toolchain
125 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
127 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
128 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
129 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
130 '-linux-gnueabihf', '-le-linux', '-uclinux']
131 for prio in range(len(priority_list)):
132 if priority_list[prio] in fname:
133 return PRIORITY_CALC + prio
134 return PRIORITY_CALC + prio
136 def GetWrapper(self, show_warning=True):
137 """Get toolchain wrapper from the setting file.
140 for name, value in bsettings.GetItems('toolchain-wrapper'):
142 print("Warning: Wrapper not found")
148 def MakeEnvironment(self, full_path):
149 """Returns an environment for using the toolchain.
151 Thie takes the current environment and adds CROSS_COMPILE so that
152 the tool chain will operate correctly. This also disables localized
153 output and possibly unicode encoded output of all build tools by
157 full_path: Return the full path in CROSS_COMPILE and don't set
160 Dict containing the environemnt to use. This is based on the current
161 environment, with changes as needed to CROSS_COMPILE, PATH and
164 env = dict(os.environ)
165 wrapper = self.GetWrapper()
167 if self.override_toolchain:
168 # We'll use MakeArgs() to provide this
171 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
173 env['CROSS_COMPILE'] = wrapper + self.cross
174 env['PATH'] = self.path + ':' + env['PATH']
181 """Create the 'make' arguments for a toolchain
183 This is only used when the toolchain is being overridden. Since the
184 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
185 environment (and MakeEnvironment()) to override these values. This
186 function returns the arguments to accomplish this.
189 List of arguments to pass to 'make'
191 if self.override_toolchain:
192 return ['HOSTCC=%s' % self.override_toolchain,
193 'CC=%s' % self.override_toolchain]
198 """Manage a list of toolchains for building U-Boot
200 We select one toolchain for each architecture type
203 toolchains: Dict of Toolchain objects, keyed by architecture name
204 prefixes: Dict of prefixes to check, keyed by architecture. This can
205 be a full path and toolchain prefix, for example
206 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
207 something on the search path, for example
208 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
209 paths: List of paths to check for toolchains (may contain wildcards)
212 def __init__(self, override_toolchain=None):
216 self.override_toolchain = override_toolchain
217 self._make_flags = dict(bsettings.GetItems('make-flags'))
219 def GetPathList(self, show_warning=True):
220 """Get a list of available toolchain paths
223 show_warning: True to show a warning if there are no tool chains.
226 List of strings, each a path to a toolchain mentioned in the
227 [toolchain] section of the settings file.
229 toolchains = bsettings.GetItems('toolchain')
230 if show_warning and not toolchains:
231 print(("Warning: No tool chains. Please run 'buildman "
232 "--fetch-arch all' to download all available toolchains, or "
233 "add a [toolchain] section to your buildman config file "
234 "%s. See README for details" %
235 bsettings.config_fname))
238 for name, value in toolchains:
240 paths += glob.glob(value)
245 def GetSettings(self, show_warning=True):
246 """Get toolchain settings from the settings file.
249 show_warning: True to show a warning if there are no tool chains.
251 self.prefixes = bsettings.GetItems('toolchain-prefix')
252 self.paths += self.GetPathList(show_warning)
254 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
256 """Add a toolchain to our list
258 We select the given toolchain as our preferred one for its
259 architecture if it is a higher priority than the others.
262 fname: Filename of toolchain's gcc driver
263 test: True to run the toolchain to test it
264 priority: Priority to use for this toolchain
265 arch: Toolchain architecture, or None if not known
267 toolchain = Toolchain(fname, test, verbose, priority, arch,
268 self.override_toolchain)
269 add_it = toolchain.ok
270 if toolchain.arch in self.toolchains:
271 add_it = (toolchain.priority <
272 self.toolchains[toolchain.arch].priority)
274 self.toolchains[toolchain.arch] = toolchain
276 print(("Toolchain '%s' at priority %d will be ignored because "
277 "another toolchain for arch '%s' has priority %d" %
278 (toolchain.gcc, toolchain.priority, toolchain.arch,
279 self.toolchains[toolchain.arch].priority)))
281 def ScanPath(self, path, verbose):
282 """Scan a path for a valid toolchain
286 verbose: True to print out progress information
288 Filename of C compiler if found, else None
291 for subdir in ['.', 'bin', 'usr/bin']:
292 dirname = os.path.join(path, subdir)
293 if verbose: print(" - looking in '%s'" % dirname)
294 for fname in glob.glob(dirname + '/*gcc'):
295 if verbose: print(" - found '%s'" % fname)
299 def ScanPathEnv(self, fname):
300 """Scan the PATH environment variable for a given filename.
303 fname: Filename to scan for
305 List of matching pathanames, or [] if none
308 for path in os.environ["PATH"].split(os.pathsep):
309 path = path.strip('"')
310 pathname = os.path.join(path, fname)
311 if os.path.exists(pathname):
312 pathname_list.append(pathname)
315 def Scan(self, verbose):
316 """Scan for available toolchains and select the best for each arch.
318 We look for all the toolchains we can file, figure out the
319 architecture for each, and whether it works. Then we select the
320 highest priority toolchain for each arch.
323 verbose: True to print out progress information
325 if verbose: print('Scanning for tool chains')
326 for name, value in self.prefixes:
327 if verbose: print(" - scanning prefix '%s'" % value)
328 if os.path.exists(value):
329 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
331 fname = value + 'gcc'
332 if os.path.exists(fname):
333 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
335 fname_list = self.ScanPathEnv(fname)
337 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
339 raise ValueError("No tool chain found for prefix '%s'" %
341 for path in self.paths:
342 if verbose: print(" - scanning path '%s'" % path)
343 fnames = self.ScanPath(path, verbose)
345 self.Add(fname, True, verbose)
348 """List out the selected toolchains for each architecture"""
349 col = terminal.Color()
350 print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
351 len(self.toolchains)))
352 if len(self.toolchains):
353 for key, value in sorted(self.toolchains.items()):
354 print('%-10s: %s' % (key, value.gcc))
358 def Select(self, arch):
359 """Returns the toolchain for a given architecture
362 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
365 toolchain object, or None if none found
367 for tag, value in bsettings.GetItems('toolchain-alias'):
369 for alias in value.split():
370 if alias in self.toolchains:
371 return self.toolchains[alias]
373 if not arch in self.toolchains:
374 raise ValueError("No tool chain found for arch '%s'" % arch)
375 return self.toolchains[arch]
377 def ResolveReferences(self, var_dict, args):
378 """Resolve variable references in a string
380 This converts ${blah} within the string to the value of blah.
381 This function works recursively.
384 var_dict: Dictionary containing variables and their values
385 args: String containing make arguments
389 >>> bsettings.Setup()
390 >>> tcs = Toolchains()
391 >>> tcs.Add('fred', False)
392 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
394 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
396 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
397 'this=OBLIQUE_setfi2ndrstnd'
399 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
402 m = re_var.search(args)
405 lookup = m.group(0)[2:-1]
406 value = var_dict.get(lookup, '')
407 args = args[:m.start(0)] + value + args[m.end(0):]
410 def GetMakeArguments(self, board):
411 """Returns 'make' arguments for a given board
413 The flags are in a section called 'make-flags'. Flags are named
414 after the target they represent, for example snapper9260=TESTING=1
415 will pass TESTING=1 to make when building the snapper9260 board.
417 References to other boards can be added in the string also. For
421 at91-boards=ENABLE_AT91_TEST=1
422 snapper9260=${at91-boards} BUILD_TAG=442
423 snapper9g45=${at91-boards} BUILD_TAG=443
425 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
426 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
428 A special 'target' variable is set to the board target.
431 board: Board object for the board to check.
433 'make' flags for that board, or '' if none
435 self._make_flags['target'] = board.target
436 arg_str = self.ResolveReferences(self._make_flags,
437 self._make_flags.get(board.target, ''))
438 args = arg_str.split(' ')
447 def LocateArchUrl(self, fetch_arch):
448 """Find a toolchain available online
450 Look in standard places for available toolchains. At present the
451 only standard place is at kernel.org.
454 arch: Architecture to look for, or 'list' for all
456 If fetch_arch is 'list', a tuple:
457 Machine architecture (e.g. x86_64)
460 URL containing this toolchain, if avaialble, else None
462 arch = command.OutputOneLine('uname', '-m')
463 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
464 versions = ['7.3.0', '6.4.0', '4.9.4']
466 for version in versions:
467 url = '%s/%s/%s/' % (base, arch, version)
468 print('Checking: %s' % url)
469 response = urllib.request.urlopen(url)
470 html = tools.ToString(response.read())
471 parser = MyHTMLParser(fetch_arch)
473 if fetch_arch == 'list':
474 links += parser.links
475 elif parser.arch_link:
476 return url + parser.arch_link
477 if fetch_arch == 'list':
481 def Download(self, url):
482 """Download a file to a temporary directory
488 Temporary directory name
489 Full path to the downloaded archive file in that directory,
490 or None if there was an error while downloading
492 print('Downloading: %s' % url)
493 leaf = url.split('/')[-1]
494 tmpdir = tempfile.mkdtemp('.buildman')
495 response = urllib.request.urlopen(url)
496 fname = os.path.join(tmpdir, leaf)
497 fd = open(fname, 'wb')
498 meta = response.info()
499 size = int(meta.get('Content-Length'))
504 # Read the file in chunks and show progress as we go
506 buffer = response.read(block_size)
508 print(chr(8) * (len(status) + 1), '\r', end=' ')
513 status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024,
515 status = status + chr(8) * (len(status) + 1)
516 print(status, end=' ')
520 print('Error, failed to download')
525 def Unpack(self, fname, dest):
529 fname: Filename to unpack
530 dest: Destination directory
532 Directory name of the first entry in the archive, without the
535 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
536 dirs = stdout.splitlines()[1].split('/')[:2]
537 return '/'.join(dirs)
539 def TestSettingsHasPath(self, path):
540 """Check if buildman will find this toolchain
543 True if the path is in settings, False if not
545 paths = self.GetPathList(False)
549 """List architectures with available toolchains to download"""
550 host_arch, archives = self.LocateArchUrl('list')
551 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
553 for archive in archives:
554 # Remove the host architecture from the start
555 arch = re_arch.match(archive[len(host_arch):])
557 if arch.group(1) != '2.0' and arch.group(1) != '64':
558 arch_set.add(arch.group(1))
559 return sorted(arch_set)
561 def FetchAndInstall(self, arch):
562 """Fetch and install a new toolchain
565 Architecture to fetch, or 'list' to list
567 # Fist get the URL for this architecture
568 col = terminal.Color()
569 print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
570 url = self.LocateArchUrl(arch)
572 print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
575 home = os.environ['HOME']
576 dest = os.path.join(home, '.buildman-toolchains')
577 if not os.path.exists(dest):
580 # Download the tar file for this toolchain and unpack it
581 tmpdir, tarfile = self.Download(url)
584 print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
586 path = self.Unpack(tarfile, dest)
591 # Check that the toolchain works
592 print(col.Color(col.GREEN, 'Testing'))
593 dirpath = os.path.join(dest, path)
594 compiler_fname_list = self.ScanPath(dirpath, True)
595 if not compiler_fname_list:
596 print('Could not locate C compiler - fetch failed.')
598 if len(compiler_fname_list) != 1:
599 print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
600 ', '.join(compiler_fname_list)))
601 toolchain = Toolchain(compiler_fname_list[0], True, True)
603 # Make sure that it will be found by buildman
604 if not self.TestSettingsHasPath(dirpath):
605 print(("Adding 'download' to config file '%s'" %
606 bsettings.config_fname))
607 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)