1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2012 The Chromium OS Authors.
7 from html.parser import HTMLParser
11 import urllib.request, urllib.error, urllib.parse
18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = list(range(4))
21 (VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
23 # Simple class to collect links from a page
24 class MyHTMLParser(HTMLParser):
25 def __init__(self, arch):
26 """Create a new parser
28 After the parser runs, self.links will be set to a list of the links
29 to .xz archives found in the page, and self.arch_link will be set to
30 the one for the given architecture (or None if not found).
33 arch: Architecture to search for
35 HTMLParser.__init__(self)
38 self.re_arch = re.compile('[-_]%s-' % arch)
40 def handle_starttag(self, tag, attrs):
42 for tag, value in attrs:
44 if value and value.endswith('.xz'):
45 self.links.append(value)
46 if self.re_arch.search(value):
47 self.arch_link = value
54 gcc: Full path to C compiler
55 path: Directory path containing C compiler
56 cross: Cross compile string, e.g. 'arm-linux-'
57 arch: Architecture of toolchain as determined from the first
58 component of the filename. E.g. arm-linux-gcc becomes arm
59 priority: Toolchain priority (0=highest, 20=lowest)
60 override_toolchain: Toolchain to use for sandbox, overriding the normal
63 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
64 arch=None, override_toolchain=None):
65 """Create a new toolchain object.
68 fname: Filename of the gcc component
69 test: True to run the toolchain to test it
70 verbose: True to print out the information
71 priority: Priority to use for this toolchain, or PRIORITY_CALC to
75 self.path = os.path.dirname(fname)
76 self.override_toolchain = override_toolchain
78 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
79 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
80 basename = os.path.basename(fname)
81 pos = basename.rfind('-')
82 self.cross = basename[:pos + 1] if pos != -1 else ''
84 # The architecture is the first part of the name
85 pos = self.cross.find('-')
89 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
90 if self.arch == 'sandbox' and override_toolchain:
91 self.gcc = override_toolchain
93 env = self.MakeEnvironment(False)
95 # As a basic sanity check, run the C compiler with --version
96 cmd = [fname, '--version']
97 if priority == PRIORITY_CALC:
98 self.priority = self.GetPriority(fname)
100 self.priority = priority
102 result = command.RunPipe([cmd], capture=True, env=env,
103 raise_on_error=False)
104 self.ok = result.return_code == 0
106 print('Tool chain test: ', end=' ')
108 print("OK, arch='%s', priority %d" % (self.arch,
112 print('Command: ', cmd)
118 def GetPriority(self, fname):
119 """Return the priority of the toolchain.
121 Toolchains are ranked according to their suitability by their
125 fname: Filename of toolchain
127 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
129 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
130 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
131 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
132 '-linux-gnueabihf', '-le-linux', '-uclinux']
133 for prio in range(len(priority_list)):
134 if priority_list[prio] in fname:
135 return PRIORITY_CALC + prio
136 return PRIORITY_CALC + prio
138 def GetWrapper(self, show_warning=True):
139 """Get toolchain wrapper from the setting file.
142 for name, value in bsettings.GetItems('toolchain-wrapper'):
144 print("Warning: Wrapper not found")
150 def GetEnvArgs(self, which):
151 """Get an environment variable/args value based on the the toolchain
154 which: VAR_... value to get
157 Value of that environment variable or arguments
159 wrapper = self.GetWrapper()
160 if which == VAR_CROSS_COMPILE:
161 return wrapper + os.path.join(self.path, self.cross)
162 elif which == VAR_PATH:
164 elif which == VAR_ARCH:
166 elif which == VAR_MAKE_ARGS:
167 args = self.MakeArgs()
169 return ' '.join(args)
172 raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
174 def MakeEnvironment(self, full_path):
175 """Returns an environment for using the toolchain.
177 Thie takes the current environment and adds CROSS_COMPILE so that
178 the tool chain will operate correctly. This also disables localized
179 output and possibly unicode encoded output of all build tools by
183 full_path: Return the full path in CROSS_COMPILE and don't set
186 Dict containing the environemnt to use. This is based on the current
187 environment, with changes as needed to CROSS_COMPILE, PATH and
190 env = dict(os.environ)
191 wrapper = self.GetWrapper()
193 if self.override_toolchain:
194 # We'll use MakeArgs() to provide this
197 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
199 env['CROSS_COMPILE'] = wrapper + self.cross
200 env['PATH'] = self.path + ':' + env['PATH']
207 """Create the 'make' arguments for a toolchain
209 This is only used when the toolchain is being overridden. Since the
210 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
211 environment (and MakeEnvironment()) to override these values. This
212 function returns the arguments to accomplish this.
215 List of arguments to pass to 'make'
217 if self.override_toolchain:
218 return ['HOSTCC=%s' % self.override_toolchain,
219 'CC=%s' % self.override_toolchain]
224 """Manage a list of toolchains for building U-Boot
226 We select one toolchain for each architecture type
229 toolchains: Dict of Toolchain objects, keyed by architecture name
230 prefixes: Dict of prefixes to check, keyed by architecture. This can
231 be a full path and toolchain prefix, for example
232 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
233 something on the search path, for example
234 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
235 paths: List of paths to check for toolchains (may contain wildcards)
238 def __init__(self, override_toolchain=None):
242 self.override_toolchain = override_toolchain
243 self._make_flags = dict(bsettings.GetItems('make-flags'))
245 def GetPathList(self, show_warning=True):
246 """Get a list of available toolchain paths
249 show_warning: True to show a warning if there are no tool chains.
252 List of strings, each a path to a toolchain mentioned in the
253 [toolchain] section of the settings file.
255 toolchains = bsettings.GetItems('toolchain')
256 if show_warning and not toolchains:
257 print(("Warning: No tool chains. Please run 'buildman "
258 "--fetch-arch all' to download all available toolchains, or "
259 "add a [toolchain] section to your buildman config file "
260 "%s. See README for details" %
261 bsettings.config_fname))
264 for name, value in toolchains:
266 paths += glob.glob(value)
271 def GetSettings(self, show_warning=True):
272 """Get toolchain settings from the settings file.
275 show_warning: True to show a warning if there are no tool chains.
277 self.prefixes = bsettings.GetItems('toolchain-prefix')
278 self.paths += self.GetPathList(show_warning)
280 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
282 """Add a toolchain to our list
284 We select the given toolchain as our preferred one for its
285 architecture if it is a higher priority than the others.
288 fname: Filename of toolchain's gcc driver
289 test: True to run the toolchain to test it
290 priority: Priority to use for this toolchain
291 arch: Toolchain architecture, or None if not known
293 toolchain = Toolchain(fname, test, verbose, priority, arch,
294 self.override_toolchain)
295 add_it = toolchain.ok
296 if toolchain.arch in self.toolchains:
297 add_it = (toolchain.priority <
298 self.toolchains[toolchain.arch].priority)
300 self.toolchains[toolchain.arch] = toolchain
302 print(("Toolchain '%s' at priority %d will be ignored because "
303 "another toolchain for arch '%s' has priority %d" %
304 (toolchain.gcc, toolchain.priority, toolchain.arch,
305 self.toolchains[toolchain.arch].priority)))
307 def ScanPath(self, path, verbose):
308 """Scan a path for a valid toolchain
312 verbose: True to print out progress information
314 Filename of C compiler if found, else None
317 for subdir in ['.', 'bin', 'usr/bin']:
318 dirname = os.path.join(path, subdir)
319 if verbose: print(" - looking in '%s'" % dirname)
320 for fname in glob.glob(dirname + '/*gcc'):
321 if verbose: print(" - found '%s'" % fname)
325 def ScanPathEnv(self, fname):
326 """Scan the PATH environment variable for a given filename.
329 fname: Filename to scan for
331 List of matching pathanames, or [] if none
334 for path in os.environ["PATH"].split(os.pathsep):
335 path = path.strip('"')
336 pathname = os.path.join(path, fname)
337 if os.path.exists(pathname):
338 pathname_list.append(pathname)
341 def Scan(self, verbose):
342 """Scan for available toolchains and select the best for each arch.
344 We look for all the toolchains we can file, figure out the
345 architecture for each, and whether it works. Then we select the
346 highest priority toolchain for each arch.
349 verbose: True to print out progress information
351 if verbose: print('Scanning for tool chains')
352 for name, value in self.prefixes:
353 if verbose: print(" - scanning prefix '%s'" % value)
354 if os.path.exists(value):
355 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
357 fname = value + 'gcc'
358 if os.path.exists(fname):
359 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
361 fname_list = self.ScanPathEnv(fname)
363 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
365 raise ValueError("No tool chain found for prefix '%s'" %
367 for path in self.paths:
368 if verbose: print(" - scanning path '%s'" % path)
369 fnames = self.ScanPath(path, verbose)
371 self.Add(fname, True, verbose)
374 """List out the selected toolchains for each architecture"""
375 col = terminal.Color()
376 print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
377 len(self.toolchains)))
378 if len(self.toolchains):
379 for key, value in sorted(self.toolchains.items()):
380 print('%-10s: %s' % (key, value.gcc))
384 def Select(self, arch):
385 """Returns the toolchain for a given architecture
388 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
391 toolchain object, or None if none found
393 for tag, value in bsettings.GetItems('toolchain-alias'):
395 for alias in value.split():
396 if alias in self.toolchains:
397 return self.toolchains[alias]
399 if not arch in self.toolchains:
400 raise ValueError("No tool chain found for arch '%s'" % arch)
401 return self.toolchains[arch]
403 def ResolveReferences(self, var_dict, args):
404 """Resolve variable references in a string
406 This converts ${blah} within the string to the value of blah.
407 This function works recursively.
410 var_dict: Dictionary containing variables and their values
411 args: String containing make arguments
415 >>> bsettings.Setup()
416 >>> tcs = Toolchains()
417 >>> tcs.Add('fred', False)
418 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
420 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
422 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
423 'this=OBLIQUE_setfi2ndrstnd'
425 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
428 m = re_var.search(args)
431 lookup = m.group(0)[2:-1]
432 value = var_dict.get(lookup, '')
433 args = args[:m.start(0)] + value + args[m.end(0):]
436 def GetMakeArguments(self, board):
437 """Returns 'make' arguments for a given board
439 The flags are in a section called 'make-flags'. Flags are named
440 after the target they represent, for example snapper9260=TESTING=1
441 will pass TESTING=1 to make when building the snapper9260 board.
443 References to other boards can be added in the string also. For
447 at91-boards=ENABLE_AT91_TEST=1
448 snapper9260=${at91-boards} BUILD_TAG=442
449 snapper9g45=${at91-boards} BUILD_TAG=443
451 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
452 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
454 A special 'target' variable is set to the board target.
457 board: Board object for the board to check.
459 'make' flags for that board, or '' if none
461 self._make_flags['target'] = board.target
462 arg_str = self.ResolveReferences(self._make_flags,
463 self._make_flags.get(board.target, ''))
464 args = re.findall("(?:\".*?\"|\S)+", arg_str)
467 args[i] = args[i].replace('"', '')
474 def LocateArchUrl(self, fetch_arch):
475 """Find a toolchain available online
477 Look in standard places for available toolchains. At present the
478 only standard place is at kernel.org.
481 arch: Architecture to look for, or 'list' for all
483 If fetch_arch is 'list', a tuple:
484 Machine architecture (e.g. x86_64)
487 URL containing this toolchain, if avaialble, else None
489 arch = command.OutputOneLine('uname', '-m')
490 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
491 versions = ['7.3.0', '6.4.0', '4.9.4']
493 for version in versions:
494 url = '%s/%s/%s/' % (base, arch, version)
495 print('Checking: %s' % url)
496 response = urllib.request.urlopen(url)
497 html = tools.ToString(response.read())
498 parser = MyHTMLParser(fetch_arch)
500 if fetch_arch == 'list':
501 links += parser.links
502 elif parser.arch_link:
503 return url + parser.arch_link
504 if fetch_arch == 'list':
508 def Download(self, url):
509 """Download a file to a temporary directory
515 Temporary directory name
516 Full path to the downloaded archive file in that directory,
517 or None if there was an error while downloading
519 print('Downloading: %s' % url)
520 leaf = url.split('/')[-1]
521 tmpdir = tempfile.mkdtemp('.buildman')
522 response = urllib.request.urlopen(url)
523 fname = os.path.join(tmpdir, leaf)
524 fd = open(fname, 'wb')
525 meta = response.info()
526 size = int(meta.get('Content-Length'))
531 # Read the file in chunks and show progress as we go
533 buffer = response.read(block_size)
535 print(chr(8) * (len(status) + 1), '\r', end=' ')
540 status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024,
542 status = status + chr(8) * (len(status) + 1)
543 print(status, end=' ')
547 print('Error, failed to download')
552 def Unpack(self, fname, dest):
556 fname: Filename to unpack
557 dest: Destination directory
559 Directory name of the first entry in the archive, without the
562 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
563 dirs = stdout.splitlines()[1].split('/')[:2]
564 return '/'.join(dirs)
566 def TestSettingsHasPath(self, path):
567 """Check if buildman will find this toolchain
570 True if the path is in settings, False if not
572 paths = self.GetPathList(False)
576 """List architectures with available toolchains to download"""
577 host_arch, archives = self.LocateArchUrl('list')
578 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
580 for archive in archives:
581 # Remove the host architecture from the start
582 arch = re_arch.match(archive[len(host_arch):])
584 if arch.group(1) != '2.0' and arch.group(1) != '64':
585 arch_set.add(arch.group(1))
586 return sorted(arch_set)
588 def FetchAndInstall(self, arch):
589 """Fetch and install a new toolchain
592 Architecture to fetch, or 'list' to list
594 # Fist get the URL for this architecture
595 col = terminal.Color()
596 print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
597 url = self.LocateArchUrl(arch)
599 print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
602 home = os.environ['HOME']
603 dest = os.path.join(home, '.buildman-toolchains')
604 if not os.path.exists(dest):
607 # Download the tar file for this toolchain and unpack it
608 tmpdir, tarfile = self.Download(url)
611 print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
613 path = self.Unpack(tarfile, dest)
618 # Check that the toolchain works
619 print(col.Color(col.GREEN, 'Testing'))
620 dirpath = os.path.join(dest, path)
621 compiler_fname_list = self.ScanPath(dirpath, True)
622 if not compiler_fname_list:
623 print('Could not locate C compiler - fetch failed.')
625 if len(compiler_fname_list) != 1:
626 print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
627 ', '.join(compiler_fname_list)))
628 toolchain = Toolchain(compiler_fname_list[0], True, True)
630 # Make sure that it will be found by buildman
631 if not self.TestSettingsHasPath(dirpath):
632 print(("Adding 'download' to config file '%s'" %
633 bsettings.config_fname))
634 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)