1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2012 The Chromium OS Authors.
7 from html.parser import HTMLParser
11 import urllib.request, urllib.error, urllib.parse
18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = list(range(4))
21 (VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
23 # Simple class to collect links from a page
24 class MyHTMLParser(HTMLParser):
25 def __init__(self, arch):
26 """Create a new parser
28 After the parser runs, self.links will be set to a list of the links
29 to .xz archives found in the page, and self.arch_link will be set to
30 the one for the given architecture (or None if not found).
33 arch: Architecture to search for
35 HTMLParser.__init__(self)
38 self.re_arch = re.compile('[-_]%s-' % arch)
40 def handle_starttag(self, tag, attrs):
42 for tag, value in attrs:
44 if value and value.endswith('.xz'):
45 self.links.append(value)
46 if self.re_arch.search(value):
47 self.arch_link = value
54 gcc: Full path to C compiler
55 path: Directory path containing C compiler
56 cross: Cross compile string, e.g. 'arm-linux-'
57 arch: Architecture of toolchain as determined from the first
58 component of the filename. E.g. arm-linux-gcc becomes arm
59 priority: Toolchain priority (0=highest, 20=lowest)
60 override_toolchain: Toolchain to use for sandbox, overriding the normal
63 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
64 arch=None, override_toolchain=None):
65 """Create a new toolchain object.
68 fname: Filename of the gcc component
69 test: True to run the toolchain to test it
70 verbose: True to print out the information
71 priority: Priority to use for this toolchain, or PRIORITY_CALC to
75 self.path = os.path.dirname(fname)
76 self.override_toolchain = override_toolchain
78 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
79 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
80 basename = os.path.basename(fname)
81 pos = basename.rfind('-')
82 self.cross = basename[:pos + 1] if pos != -1 else ''
84 # The architecture is the first part of the name
85 pos = self.cross.find('-')
89 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
90 if self.arch == 'sandbox' and override_toolchain:
91 self.gcc = override_toolchain
93 env = self.MakeEnvironment(False)
95 # As a basic sanity check, run the C compiler with --version
96 cmd = [fname, '--version']
97 if priority == PRIORITY_CALC:
98 self.priority = self.GetPriority(fname)
100 self.priority = priority
102 result = command.RunPipe([cmd], capture=True, env=env,
103 raise_on_error=False)
104 self.ok = result.return_code == 0
106 print('Tool chain test: ', end=' ')
108 print("OK, arch='%s', priority %d" % (self.arch,
112 print('Command: ', cmd)
118 def GetPriority(self, fname):
119 """Return the priority of the toolchain.
121 Toolchains are ranked according to their suitability by their
125 fname: Filename of toolchain
127 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
129 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
130 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
131 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
132 '-linux-gnueabihf', '-le-linux', '-uclinux']
133 for prio in range(len(priority_list)):
134 if priority_list[prio] in fname:
135 return PRIORITY_CALC + prio
136 return PRIORITY_CALC + prio
138 def GetWrapper(self, show_warning=True):
139 """Get toolchain wrapper from the setting file.
142 for name, value in bsettings.GetItems('toolchain-wrapper'):
144 print("Warning: Wrapper not found")
150 def GetEnvArgs(self, which):
151 """Get an environment variable/args value based on the the toolchain
154 which: VAR_... value to get
157 Value of that environment variable or arguments
159 wrapper = self.GetWrapper()
160 if which == VAR_CROSS_COMPILE:
161 return wrapper + os.path.join(self.path, self.cross)
162 elif which == VAR_PATH:
164 elif which == VAR_ARCH:
166 elif which == VAR_MAKE_ARGS:
167 args = self.MakeArgs()
169 return ' '.join(args)
172 raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
174 def MakeEnvironment(self, full_path):
175 """Returns an environment for using the toolchain.
177 Thie takes the current environment and adds CROSS_COMPILE so that
178 the tool chain will operate correctly. This also disables localized
179 output and possibly unicode encoded output of all build tools by
183 full_path: Return the full path in CROSS_COMPILE and don't set
186 Dict containing the environemnt to use. This is based on the current
187 environment, with changes as needed to CROSS_COMPILE, PATH and
190 env = dict(os.environ)
191 wrapper = self.GetWrapper()
193 if self.override_toolchain:
194 # We'll use MakeArgs() to provide this
197 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
199 env['CROSS_COMPILE'] = wrapper + self.cross
200 env['PATH'] = self.path + ':' + env['PATH']
207 """Create the 'make' arguments for a toolchain
209 This is only used when the toolchain is being overridden. Since the
210 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
211 environment (and MakeEnvironment()) to override these values. This
212 function returns the arguments to accomplish this.
215 List of arguments to pass to 'make'
217 if self.override_toolchain:
218 return ['HOSTCC=%s' % self.override_toolchain,
219 'CC=%s' % self.override_toolchain]
224 """Manage a list of toolchains for building U-Boot
226 We select one toolchain for each architecture type
229 toolchains: Dict of Toolchain objects, keyed by architecture name
230 prefixes: Dict of prefixes to check, keyed by architecture. This can
231 be a full path and toolchain prefix, for example
232 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
233 something on the search path, for example
234 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
235 paths: List of paths to check for toolchains (may contain wildcards)
238 def __init__(self, override_toolchain=None):
242 self.override_toolchain = override_toolchain
243 self._make_flags = dict(bsettings.GetItems('make-flags'))
245 def GetPathList(self, show_warning=True):
246 """Get a list of available toolchain paths
249 show_warning: True to show a warning if there are no tool chains.
252 List of strings, each a path to a toolchain mentioned in the
253 [toolchain] section of the settings file.
255 toolchains = bsettings.GetItems('toolchain')
256 if show_warning and not toolchains:
257 print(("Warning: No tool chains. Please run 'buildman "
258 "--fetch-arch all' to download all available toolchains, or "
259 "add a [toolchain] section to your buildman config file "
260 "%s. See README for details" %
261 bsettings.config_fname))
264 for name, value in toolchains:
266 paths += glob.glob(value)
271 def GetSettings(self, show_warning=True):
272 """Get toolchain settings from the settings file.
275 show_warning: True to show a warning if there are no tool chains.
277 self.prefixes = bsettings.GetItems('toolchain-prefix')
278 self.paths += self.GetPathList(show_warning)
280 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
282 """Add a toolchain to our list
284 We select the given toolchain as our preferred one for its
285 architecture if it is a higher priority than the others.
288 fname: Filename of toolchain's gcc driver
289 test: True to run the toolchain to test it
290 priority: Priority to use for this toolchain
291 arch: Toolchain architecture, or None if not known
293 toolchain = Toolchain(fname, test, verbose, priority, arch,
294 self.override_toolchain)
295 add_it = toolchain.ok
296 if toolchain.arch in self.toolchains:
297 add_it = (toolchain.priority <
298 self.toolchains[toolchain.arch].priority)
300 self.toolchains[toolchain.arch] = toolchain
302 print(("Toolchain '%s' at priority %d will be ignored because "
303 "another toolchain for arch '%s' has priority %d" %
304 (toolchain.gcc, toolchain.priority, toolchain.arch,
305 self.toolchains[toolchain.arch].priority)))
307 def ScanPath(self, path, verbose):
308 """Scan a path for a valid toolchain
312 verbose: True to print out progress information
314 Filename of C compiler if found, else None
317 for subdir in ['.', 'bin', 'usr/bin']:
318 dirname = os.path.join(path, subdir)
319 if verbose: print(" - looking in '%s'" % dirname)
320 for fname in glob.glob(dirname + '/*gcc'):
321 if verbose: print(" - found '%s'" % fname)
325 def ScanPathEnv(self, fname):
326 """Scan the PATH environment variable for a given filename.
329 fname: Filename to scan for
331 List of matching pathanames, or [] if none
334 for path in os.environ["PATH"].split(os.pathsep):
335 path = path.strip('"')
336 pathname = os.path.join(path, fname)
337 if os.path.exists(pathname):
338 pathname_list.append(pathname)
341 def Scan(self, verbose):
342 """Scan for available toolchains and select the best for each arch.
344 We look for all the toolchains we can file, figure out the
345 architecture for each, and whether it works. Then we select the
346 highest priority toolchain for each arch.
349 verbose: True to print out progress information
351 if verbose: print('Scanning for tool chains')
352 for name, value in self.prefixes:
353 if verbose: print(" - scanning prefix '%s'" % value)
354 if os.path.exists(value):
355 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
357 fname = value + 'gcc'
358 if os.path.exists(fname):
359 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
361 fname_list = self.ScanPathEnv(fname)
363 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
365 raise ValueError("No tool chain found for prefix '%s'" %
367 for path in self.paths:
368 if verbose: print(" - scanning path '%s'" % path)
369 fnames = self.ScanPath(path, verbose)
371 self.Add(fname, True, verbose)
374 """List out the selected toolchains for each architecture"""
375 col = terminal.Color()
376 print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
377 len(self.toolchains)))
378 if len(self.toolchains):
379 for key, value in sorted(self.toolchains.items()):
380 print('%-10s: %s' % (key, value.gcc))
384 def Select(self, arch):
385 """Returns the toolchain for a given architecture
388 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
391 toolchain object, or None if none found
393 for tag, value in bsettings.GetItems('toolchain-alias'):
395 for alias in value.split():
396 if alias in self.toolchains:
397 return self.toolchains[alias]
399 if not arch in self.toolchains:
400 raise ValueError("No tool chain found for arch '%s'" % arch)
401 return self.toolchains[arch]
403 def ResolveReferences(self, var_dict, args):
404 """Resolve variable references in a string
406 This converts ${blah} within the string to the value of blah.
407 This function works recursively.
410 var_dict: Dictionary containing variables and their values
411 args: String containing make arguments
415 >>> bsettings.Setup()
416 >>> tcs = Toolchains()
417 >>> tcs.Add('fred', False)
418 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
420 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
422 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
423 'this=OBLIQUE_setfi2ndrstnd'
425 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
428 m = re_var.search(args)
431 lookup = m.group(0)[2:-1]
432 value = var_dict.get(lookup, '')
433 args = args[:m.start(0)] + value + args[m.end(0):]
436 def GetMakeArguments(self, board):
437 """Returns 'make' arguments for a given board
439 The flags are in a section called 'make-flags'. Flags are named
440 after the target they represent, for example snapper9260=TESTING=1
441 will pass TESTING=1 to make when building the snapper9260 board.
443 References to other boards can be added in the string also. For
447 at91-boards=ENABLE_AT91_TEST=1
448 snapper9260=${at91-boards} BUILD_TAG=442
449 snapper9g45=${at91-boards} BUILD_TAG=443
451 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
452 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
454 A special 'target' variable is set to the board target.
457 board: Board object for the board to check.
459 'make' flags for that board, or '' if none
461 self._make_flags['target'] = board.target
462 arg_str = self.ResolveReferences(self._make_flags,
463 self._make_flags.get(board.target, ''))
464 args = re.findall("(?:\".*?\"|\S)+", arg_str)
467 args[i] = args[i].replace('"', '')
474 def LocateArchUrl(self, fetch_arch):
475 """Find a toolchain available online
477 Look in standard places for available toolchains. At present the
478 only standard place is at kernel.org.
481 arch: Architecture to look for, or 'list' for all
483 If fetch_arch is 'list', a tuple:
484 Machine architecture (e.g. x86_64)
487 URL containing this toolchain, if avaialble, else None
489 arch = command.OutputOneLine('uname', '-m')
490 if arch == 'aarch64':
492 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
493 versions = ['9.2.0', '7.3.0', '6.4.0', '4.9.4']
495 for version in versions:
496 url = '%s/%s/%s/' % (base, arch, version)
497 print('Checking: %s' % url)
498 response = urllib.request.urlopen(url)
499 html = tools.ToString(response.read())
500 parser = MyHTMLParser(fetch_arch)
502 if fetch_arch == 'list':
503 links += parser.links
504 elif parser.arch_link:
505 return url + parser.arch_link
506 if fetch_arch == 'list':
510 def Download(self, url):
511 """Download a file to a temporary directory
517 Temporary directory name
518 Full path to the downloaded archive file in that directory,
519 or None if there was an error while downloading
521 print('Downloading: %s' % url)
522 leaf = url.split('/')[-1]
523 tmpdir = tempfile.mkdtemp('.buildman')
524 response = urllib.request.urlopen(url)
525 fname = os.path.join(tmpdir, leaf)
526 fd = open(fname, 'wb')
527 meta = response.info()
528 size = int(meta.get('Content-Length'))
533 # Read the file in chunks and show progress as we go
535 buffer = response.read(block_size)
537 print(chr(8) * (len(status) + 1), '\r', end=' ')
542 status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024,
544 status = status + chr(8) * (len(status) + 1)
545 print(status, end=' ')
549 print('Error, failed to download')
554 def Unpack(self, fname, dest):
558 fname: Filename to unpack
559 dest: Destination directory
561 Directory name of the first entry in the archive, without the
564 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
565 dirs = stdout.splitlines()[1].split('/')[:2]
566 return '/'.join(dirs)
568 def TestSettingsHasPath(self, path):
569 """Check if buildman will find this toolchain
572 True if the path is in settings, False if not
574 paths = self.GetPathList(False)
578 """List architectures with available toolchains to download"""
579 host_arch, archives = self.LocateArchUrl('list')
580 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
582 for archive in archives:
583 # Remove the host architecture from the start
584 arch = re_arch.match(archive[len(host_arch):])
586 if arch.group(1) != '2.0' and arch.group(1) != '64':
587 arch_set.add(arch.group(1))
588 return sorted(arch_set)
590 def FetchAndInstall(self, arch):
591 """Fetch and install a new toolchain
594 Architecture to fetch, or 'list' to list
596 # Fist get the URL for this architecture
597 col = terminal.Color()
598 print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
599 url = self.LocateArchUrl(arch)
601 print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
604 home = os.environ['HOME']
605 dest = os.path.join(home, '.buildman-toolchains')
606 if not os.path.exists(dest):
609 # Download the tar file for this toolchain and unpack it
610 tmpdir, tarfile = self.Download(url)
613 print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
615 path = self.Unpack(tarfile, dest)
620 # Check that the toolchain works
621 print(col.Color(col.GREEN, 'Testing'))
622 dirpath = os.path.join(dest, path)
623 compiler_fname_list = self.ScanPath(dirpath, True)
624 if not compiler_fname_list:
625 print('Could not locate C compiler - fetch failed.')
627 if len(compiler_fname_list) != 1:
628 print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
629 ', '.join(compiler_fname_list)))
630 toolchain = Toolchain(compiler_fname_list[0], True, True)
632 # Make sure that it will be found by buildman
633 if not self.TestSettingsHasPath(dirpath):
634 print(("Adding 'download' to config file '%s'" %
635 bsettings.config_fname))
636 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)