X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=tools%2Fbuildman%2Ftoolchain.py;h=4456a805c742d0505500f8c5ea27803f794d660a;hb=205254eab8050eb0f6cbb0e0ff6e2b54fa04b853;hp=e33e10532ee9171897426f8e817bb112396083dc;hpb=b939689c7b87773c44275a578ffc8674a867e39d;p=oweals%2Fu-boot.git diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index e33e10532e..4456a805c7 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -1,18 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0+ # Copyright (c) 2012 The Chromium OS Authors. # -# SPDX-License-Identifier: GPL-2.0+ -# import re import glob -from HTMLParser import HTMLParser +from html.parser import HTMLParser import os import sys import tempfile -import urllib2 +import urllib.request, urllib.error, urllib.parse import bsettings import command +import terminal +import tools + +(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, + PRIORITY_CALC) = list(range(4)) + +(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4) # Simple class to collect links from a page class MyHTMLParser(HTMLParser): @@ -29,7 +35,7 @@ class MyHTMLParser(HTMLParser): HTMLParser.__init__(self) self.arch_link = None self.links = [] - self._match = '_%s-' % arch + self.re_arch = re.compile('[-_]%s-' % arch) def handle_starttag(self, tag, attrs): if tag == 'a': @@ -37,7 +43,7 @@ class MyHTMLParser(HTMLParser): if tag == 'href': if value and value.endswith('.xz'): self.links.append(value) - if self._match in value: + if self.re_arch.search(value): self.arch_link = value @@ -50,16 +56,24 @@ class Toolchain: cross: Cross compile string, e.g. 'arm-linux-' arch: Architecture of toolchain as determined from the first component of the filename. E.g. arm-linux-gcc becomes arm + priority: Toolchain priority (0=highest, 20=lowest) + override_toolchain: Toolchain to use for sandbox, overriding the normal + one """ - def __init__(self, fname, test, verbose=False): + def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, + arch=None, override_toolchain=None): """Create a new toolchain object. Args: fname: Filename of the gcc component test: True to run the toolchain to test it + verbose: True to print out the information + priority: Priority to use for this toolchain, or PRIORITY_CALC to + calculate it """ self.gcc = fname self.path = os.path.dirname(fname) + self.override_toolchain = override_toolchain # Find the CROSS_COMPILE prefix to use for U-Boot. For example, # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. @@ -69,28 +83,37 @@ class Toolchain: # The architecture is the first part of the name pos = self.cross.find('-') - self.arch = self.cross[:pos] if pos != -1 else 'sandbox' + if arch: + self.arch = arch + else: + self.arch = self.cross[:pos] if pos != -1 else 'sandbox' + if self.arch == 'sandbox' and override_toolchain: + self.gcc = override_toolchain env = self.MakeEnvironment(False) # As a basic sanity check, run the C compiler with --version cmd = [fname, '--version'] + if priority == PRIORITY_CALC: + self.priority = self.GetPriority(fname) + else: + self.priority = priority if test: result = command.RunPipe([cmd], capture=True, env=env, raise_on_error=False) self.ok = result.return_code == 0 if verbose: - print 'Tool chain test: ', + print('Tool chain test: ', end=' ') if self.ok: - print 'OK' + print("OK, arch='%s', priority %d" % (self.arch, + self.priority)) else: - print 'BAD' - print 'Command: ', cmd - print result.stdout - print result.stderr + print('BAD') + print('Command: ', cmd) + print(result.stdout) + print(result.stderr) else: self.ok = True - self.priority = self.GetPriority(fname) def GetPriority(self, fname): """Return the priority of the toolchain. @@ -101,35 +124,101 @@ class Toolchain: Args: fname: Filename of toolchain Returns: - Priority of toolchain, 0=highest, 20=lowest. + Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. """ priority_list = ['-elf', '-unknown-linux-gnu', '-linux', - '-none-linux-gnueabi', '-uclinux', '-none-eabi', - '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux'] + '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux', + '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi', + '-linux-gnueabihf', '-le-linux', '-uclinux'] for prio in range(len(priority_list)): if priority_list[prio] in fname: - return prio - return prio + return PRIORITY_CALC + prio + return PRIORITY_CALC + prio + + def GetWrapper(self, show_warning=True): + """Get toolchain wrapper from the setting file. + """ + value = '' + for name, value in bsettings.GetItems('toolchain-wrapper'): + if not value: + print("Warning: Wrapper not found") + if value: + value = value + ' ' + + return value + + def GetEnvArgs(self, which): + """Get an environment variable/args value based on the the toolchain + + Args: + which: VAR_... value to get + + Returns: + Value of that environment variable or arguments + """ + wrapper = self.GetWrapper() + if which == VAR_CROSS_COMPILE: + return wrapper + os.path.join(self.path, self.cross) + elif which == VAR_PATH: + return self.path + elif which == VAR_ARCH: + return self.arch + elif which == VAR_MAKE_ARGS: + args = self.MakeArgs() + if args: + return ' '.join(args) + return '' + else: + raise ValueError('Unknown arg to GetEnvArgs (%d)' % which) def MakeEnvironment(self, full_path): """Returns an environment for using the toolchain. Thie takes the current environment and adds CROSS_COMPILE so that - the tool chain will operate correctly. + the tool chain will operate correctly. This also disables localized + output and possibly unicode encoded output of all build tools by + adding LC_ALL=C. Args: full_path: Return the full path in CROSS_COMPILE and don't set PATH + Returns: + Dict containing the environemnt to use. This is based on the current + environment, with changes as needed to CROSS_COMPILE, PATH and + LC_ALL. """ env = dict(os.environ) - if full_path: - env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) + wrapper = self.GetWrapper() + + if self.override_toolchain: + # We'll use MakeArgs() to provide this + pass + elif full_path: + env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross) else: - env['CROSS_COMPILE'] = self.cross + env['CROSS_COMPILE'] = wrapper + self.cross env['PATH'] = self.path + ':' + env['PATH'] + env['LC_ALL'] = 'C' + return env + def MakeArgs(self): + """Create the 'make' arguments for a toolchain + + This is only used when the toolchain is being overridden. Since the + U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the + environment (and MakeEnvironment()) to override these values. This + function returns the arguments to accomplish this. + + Returns: + List of arguments to pass to 'make' + """ + if self.override_toolchain: + return ['HOSTCC=%s' % self.override_toolchain, + 'CC=%s' % self.override_toolchain] + return [] + class Toolchains: """Manage a list of toolchains for building U-Boot @@ -138,26 +227,38 @@ class Toolchains: Public members: toolchains: Dict of Toolchain objects, keyed by architecture name + prefixes: Dict of prefixes to check, keyed by architecture. This can + be a full path and toolchain prefix, for example + {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of + something on the search path, for example + {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. paths: List of paths to check for toolchains (may contain wildcards) """ - def __init__(self): + def __init__(self, override_toolchain=None): self.toolchains = {} + self.prefixes = {} self.paths = [] + self.override_toolchain = override_toolchain self._make_flags = dict(bsettings.GetItems('make-flags')) - def GetPathList(self): + def GetPathList(self, show_warning=True): """Get a list of available toolchain paths + Args: + show_warning: True to show a warning if there are no tool chains. + Returns: List of strings, each a path to a toolchain mentioned in the [toolchain] section of the settings file. """ toolchains = bsettings.GetItems('toolchain') - if not toolchains: - print ("Warning: No tool chains - please add a [toolchain] section" - " to your buildman config file %s. See README for details" % - bsettings.config_fname) + if show_warning and not toolchains: + print(("Warning: No tool chains. Please run 'buildman " + "--fetch-arch all' to download all available toolchains, or " + "add a [toolchain] section to your buildman config file " + "%s. See README for details" % + bsettings.config_fname)) paths = [] for name, value in toolchains: @@ -167,10 +268,17 @@ class Toolchains: paths.append(value) return paths - def GetSettings(self): - self.paths += self.GetPathList() + def GetSettings(self, show_warning=True): + """Get toolchain settings from the settings file. - def Add(self, fname, test=True, verbose=False): + Args: + show_warning: True to show a warning if there are no tool chains. + """ + self.prefixes = bsettings.GetItems('toolchain-prefix') + self.paths += self.GetPathList(show_warning) + + def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, + arch=None): """Add a toolchain to our list We select the given toolchain as our preferred one for its @@ -179,14 +287,22 @@ class Toolchains: Args: fname: Filename of toolchain's gcc driver test: True to run the toolchain to test it + priority: Priority to use for this toolchain + arch: Toolchain architecture, or None if not known """ - toolchain = Toolchain(fname, test, verbose) + toolchain = Toolchain(fname, test, verbose, priority, arch, + self.override_toolchain) add_it = toolchain.ok if toolchain.arch in self.toolchains: add_it = (toolchain.priority < self.toolchains[toolchain.arch].priority) if add_it: self.toolchains[toolchain.arch] = toolchain + elif verbose: + print(("Toolchain '%s' at priority %d will be ignored because " + "another toolchain for arch '%s' has priority %d" % + (toolchain.gcc, toolchain.priority, toolchain.arch, + self.toolchains[toolchain.arch].priority))) def ScanPath(self, path, verbose): """Scan a path for a valid toolchain @@ -200,12 +316,27 @@ class Toolchains: fnames = [] for subdir in ['.', 'bin', 'usr/bin']: dirname = os.path.join(path, subdir) - if verbose: print " - looking in '%s'" % dirname + if verbose: print(" - looking in '%s'" % dirname) for fname in glob.glob(dirname + '/*gcc'): - if verbose: print " - found '%s'" % fname + if verbose: print(" - found '%s'" % fname) fnames.append(fname) return fnames + def ScanPathEnv(self, fname): + """Scan the PATH environment variable for a given filename. + + Args: + fname: Filename to scan for + Returns: + List of matching pathanames, or [] if none + """ + pathname_list = [] + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + pathname = os.path.join(path, fname) + if os.path.exists(pathname): + pathname_list.append(pathname) + return pathname_list def Scan(self, verbose): """Scan for available toolchains and select the best for each arch. @@ -217,21 +348,38 @@ class Toolchains: Args: verbose: True to print out progress information """ - if verbose: print 'Scanning for tool chains' + if verbose: print('Scanning for tool chains') + for name, value in self.prefixes: + if verbose: print(" - scanning prefix '%s'" % value) + if os.path.exists(value): + self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) + continue + fname = value + 'gcc' + if os.path.exists(fname): + self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) + continue + fname_list = self.ScanPathEnv(fname) + for f in fname_list: + self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) + if not fname_list: + raise ValueError("No tool chain found for prefix '%s'" % + value) for path in self.paths: - if verbose: print " - scanning path '%s'" % path + if verbose: print(" - scanning path '%s'" % path) fnames = self.ScanPath(path, verbose) for fname in fnames: self.Add(fname, True, verbose) def List(self): """List out the selected toolchains for each architecture""" - print 'List of available toolchains (%d):' % len(self.toolchains) + col = terminal.Color() + print(col.Color(col.BLUE, 'List of available toolchains (%d):' % + len(self.toolchains))) if len(self.toolchains): - for key, value in sorted(self.toolchains.iteritems()): - print '%-10s: %s' % (key, value.gcc) + for key, value in sorted(self.toolchains.items()): + print('%-10s: %s' % (key, value.gcc)) else: - print 'None' + print('None') def Select(self, arch): """Returns the toolchain for a given architecture @@ -249,7 +397,7 @@ class Toolchains: return self.toolchains[alias] if not arch in self.toolchains: - raise ValueError, ("No tool chain found for arch '%s'" % arch) + raise ValueError("No tool chain found for arch '%s'" % arch) return self.toolchains[arch] def ResolveReferences(self, var_dict, args): @@ -313,9 +461,10 @@ class Toolchains: self._make_flags['target'] = board.target arg_str = self.ResolveReferences(self._make_flags, self._make_flags.get(board.target, '')) - args = arg_str.split(' ') + args = re.findall("(?:\".*?\"|\S)+", arg_str) i = 0 while i < len(args): + args[i] = args[i].replace('"', '') if not args[i]: del args[i] else: @@ -338,14 +487,16 @@ class Toolchains: URL containing this toolchain, if avaialble, else None """ arch = command.OutputOneLine('uname', '-m') + if arch == 'aarch64': + arch = 'arm64' base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' - versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4'] + versions = ['9.2.0', '7.3.0', '6.4.0', '4.9.4'] links = [] for version in versions: url = '%s/%s/%s/' % (base, arch, version) - print 'Checking: %s' % url - response = urllib2.urlopen(url) - html = response.read() + print('Checking: %s' % url) + response = urllib.request.urlopen(url) + html = tools.ToString(response.read()) parser = MyHTMLParser(fetch_arch) parser.feed(html) if fetch_arch == 'list': @@ -367,14 +518,14 @@ class Toolchains: Full path to the downloaded archive file in that directory, or None if there was an error while downloading """ - print "Downloading: %s" % url + print('Downloading: %s' % url) leaf = url.split('/')[-1] tmpdir = tempfile.mkdtemp('.buildman') - response = urllib2.urlopen(url) + response = urllib.request.urlopen(url) fname = os.path.join(tmpdir, leaf) fd = open(fname, 'wb') meta = response.info() - size = int(meta.getheaders("Content-Length")[0]) + size = int(meta.get('Content-Length')) done = 0 block_size = 1 << 16 status = '' @@ -383,19 +534,19 @@ class Toolchains: while True: buffer = response.read(block_size) if not buffer: - print chr(8) * (len(status) + 1), '\r', + print(chr(8) * (len(status) + 1), '\r', end=' ') break done += len(buffer) fd.write(buffer) - status = r"%10d MiB [%3d%%]" % (done / 1024 / 1024, - done * 100 / size) + status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024, + done * 100 // size) status = status + chr(8) * (len(status) + 1) - print status, + print(status, end=' ') sys.stdout.flush() fd.close() if done != size: - print 'Error, failed to download' + print('Error, failed to download') os.remove(fname) fname = None return tmpdir, fname @@ -411,27 +562,29 @@ class Toolchains: trailing / """ stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) - return stdout.splitlines()[0][:-1] + dirs = stdout.splitlines()[1].split('/')[:2] + return '/'.join(dirs) def TestSettingsHasPath(self, path): - """Check if builmand will find this toolchain + """Check if buildman will find this toolchain Returns: True if the path is in settings, False if not """ - paths = self.GetPathList() + paths = self.GetPathList(False) return path in paths def ListArchs(self): """List architectures with available toolchains to download""" host_arch, archives = self.LocateArchUrl('list') - re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') + re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*') arch_set = set() for archive in archives: # Remove the host architecture from the start arch = re_arch.match(archive[len(host_arch):]) if arch: - arch_set.add(arch.group(1)) + if arch.group(1) != '2.0' and arch.group(1) != '64': + arch_set.add(arch.group(1)) return sorted(arch_set) def FetchAndInstall(self, arch): @@ -441,10 +594,12 @@ class Toolchains: Architecture to fetch, or 'list' to list """ # Fist get the URL for this architecture + col = terminal.Color() + print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)) url = self.LocateArchUrl(arch) if not url: - print ("Cannot find toolchain for arch '%s' - use 'list' to list" % - arch) + print(("Cannot find toolchain for arch '%s' - use 'list' to list" % + arch)) return 2 home = os.environ['HOME'] dest = os.path.join(home, '.buildman-toolchains') @@ -455,30 +610,28 @@ class Toolchains: tmpdir, tarfile = self.Download(url) if not tarfile: return 1 - print 'Unpacking to: %s' % dest, + print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ') sys.stdout.flush() path = self.Unpack(tarfile, dest) os.remove(tarfile) os.rmdir(tmpdir) - print + print() # Check that the toolchain works - print 'Testing' + print(col.Color(col.GREEN, 'Testing')) dirpath = os.path.join(dest, path) compiler_fname_list = self.ScanPath(dirpath, True) if not compiler_fname_list: - print 'Could not locate C compiler - fetch failed.' + print('Could not locate C compiler - fetch failed.') return 1 if len(compiler_fname_list) != 1: - print ('Internal error, ambiguous toolchains: %s' % - (', '.join(compiler_fname))) - return 1 + print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' % + ', '.join(compiler_fname_list))) toolchain = Toolchain(compiler_fname_list[0], True, True) # Make sure that it will be found by buildman if not self.TestSettingsHasPath(dirpath): - print ("Adding 'download' to config file '%s'" % - bsettings.config_fname) - tools_dir = os.path.dirname(dirpath) - bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) + print(("Adding 'download' to config file '%s'" % + bsettings.config_fname)) + bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest) return 0