buildman: Support fetching gcc 9.2.0

[oweals/u-boot.git] / tools / buildman / toolchain.py

diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py
index 2076323d5d39825f17b0460d09ce3dd383bc50b3..4456a805c742d0505500f8c5ea27803f794d660a 100644 (file)
--- a/tools/buildman/toolchain.py
+++ b/tools/buildman/toolchain.py
@@ -1,22 +1,24 @@
+# SPDX-License-Identifier: GPL-2.0+

 # Copyright (c) 2012 The Chromium OS Authors.
 #
 # Copyright (c) 2012 The Chromium OS Authors.
 #

-# SPDX-License-Identifier:     GPL-2.0+
-#

 
 import re
 import glob
 
 import re
 import glob

-from HTMLParser import HTMLParser
+from html.parser import HTMLParser

 import os
 import sys
 import tempfile
 import os
 import sys
 import tempfile

-import urllib2
+import urllib.request, urllib.error, urllib.parse

 
 import bsettings
 import command
 import terminal
 
 import bsettings
 import command
 import terminal

+import tools

 
 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
 
 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,

-    PRIORITY_CALC) = range(4)
+    PRIORITY_CALC) = list(range(4))
+
+(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)

 
 # Simple class to collect links from a page
 class MyHTMLParser(HTMLParser):
 
 # Simple class to collect links from a page
 class MyHTMLParser(HTMLParser):


@@ -33,7 +35,7 @@ class MyHTMLParser(HTMLParser):
         HTMLParser.__init__(self)
         self.arch_link = None
         self.links = []
         HTMLParser.__init__(self)
         self.arch_link = None
         self.links = []

-        self._match = '_%s-' % arch
+        self.re_arch = re.compile('[-_]%s-' % arch)

 
     def handle_starttag(self, tag, attrs):
         if tag == 'a':
 
     def handle_starttag(self, tag, attrs):
         if tag == 'a':


@@ -41,7 +43,7 @@ class MyHTMLParser(HTMLParser):
                 if tag == 'href':
                     if value and value.endswith('.xz'):
                         self.links.append(value)
                 if tag == 'href':
                     if value and value.endswith('.xz'):
                         self.links.append(value)

-                        if self._match in value:
+                        if self.re_arch.search(value):

                             self.arch_link = value
 
 
                             self.arch_link = value
 
 


@@ -55,9 +57,11 @@ class Toolchain:
         arch: Architecture of toolchain as determined from the first
                 component of the filename. E.g. arm-linux-gcc becomes arm
         priority: Toolchain priority (0=highest, 20=lowest)
         arch: Architecture of toolchain as determined from the first
                 component of the filename. E.g. arm-linux-gcc becomes arm
         priority: Toolchain priority (0=highest, 20=lowest)

+        override_toolchain: Toolchain to use for sandbox, overriding the normal
+                one

     """
     def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
     """
     def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,

-                 arch=None):
+                 arch=None, override_toolchain=None):

         """Create a new toolchain object.
 
         Args:
         """Create a new toolchain object.
 
         Args:


@@ -69,6 +73,7 @@ class Toolchain:
         """
         self.gcc = fname
         self.path = os.path.dirname(fname)
         """
         self.gcc = fname
         self.path = os.path.dirname(fname)

+        self.override_toolchain = override_toolchain

 
         # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
         # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
 
         # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
         # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.


@@ -82,6 +87,8 @@ class Toolchain:
             self.arch = arch
         else:
             self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
             self.arch = arch
         else:
             self.arch = self.cross[:pos] if pos != -1 else 'sandbox'

+        if self.arch == 'sandbox' and override_toolchain:
+            self.gcc = override_toolchain

 
         env = self.MakeEnvironment(False)
 
 
         env = self.MakeEnvironment(False)
 


@@ -96,15 +103,15 @@ class Toolchain:
                                      raise_on_error=False)
             self.ok = result.return_code == 0
             if verbose:
                                      raise_on_error=False)
             self.ok = result.return_code == 0
             if verbose:

-                print 'Tool chain test: ',
+                print('Tool chain test: ', end=' ')

                 if self.ok:
                 if self.ok:

-                    print "OK, arch='%s', priority %d" % (self.arch,
-                                                          self.priority)
+                    print("OK, arch='%s', priority %d" % (self.arch,
+                                                          self.priority))

                 else:
                 else:

-                    print 'BAD'
-                    print 'Command: ', cmd
-                    print result.stdout
-                    print result.stderr
+                    print('BAD')
+                    print('Command: ', cmd)
+                    print(result.stdout)
+                    print(result.stderr)

         else:
             self.ok = True
 
         else:
             self.ok = True
 


@@ -131,15 +138,39 @@ class Toolchain:
     def GetWrapper(self, show_warning=True):
         """Get toolchain wrapper from the setting file.
         """
     def GetWrapper(self, show_warning=True):
         """Get toolchain wrapper from the setting file.
         """

-       value = ''
-       for name, value in bsettings.GetItems('toolchain-wrapper'):
+        value = ''
+        for name, value in bsettings.GetItems('toolchain-wrapper'):

             if not value:
             if not value:

-                print "Warning: Wrapper not found"
+                print("Warning: Wrapper not found")

         if value:
             value = value + ' '
 
         return value
 
         if value:
             value = value + ' '
 
         return value
 

+    def GetEnvArgs(self, which):
+        """Get an environment variable/args value based on the the toolchain
+
+        Args:
+            which: VAR_... value to get
+
+        Returns:
+            Value of that environment variable or arguments
+        """
+        wrapper = self.GetWrapper()
+        if which == VAR_CROSS_COMPILE:
+            return wrapper + os.path.join(self.path, self.cross)
+        elif which == VAR_PATH:
+            return self.path
+        elif which == VAR_ARCH:
+            return self.arch
+        elif which == VAR_MAKE_ARGS:
+            args = self.MakeArgs()
+            if args:
+                return ' '.join(args)
+            return ''
+        else:
+            raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
+

     def MakeEnvironment(self, full_path):
         """Returns an environment for using the toolchain.
 
     def MakeEnvironment(self, full_path):
         """Returns an environment for using the toolchain.
 


@@ -151,11 +182,18 @@ class Toolchain:
         Args:
             full_path: Return the full path in CROSS_COMPILE and don't set
                 PATH
         Args:
             full_path: Return the full path in CROSS_COMPILE and don't set
                 PATH

+        Returns:
+            Dict containing the environemnt to use. This is based on the current
+            environment, with changes as needed to CROSS_COMPILE, PATH and
+            LC_ALL.

         """
         env = dict(os.environ)
         wrapper = self.GetWrapper()
 
         """
         env = dict(os.environ)
         wrapper = self.GetWrapper()
 

-        if full_path:
+        if self.override_toolchain:
+            # We'll use MakeArgs() to provide this
+            pass
+        elif full_path:

             env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
         else:
             env['CROSS_COMPILE'] = wrapper + self.cross
             env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
         else:
             env['CROSS_COMPILE'] = wrapper + self.cross


@@ -165,6 +203,22 @@ class Toolchain:
 
         return env
 
 
         return env
 

+    def MakeArgs(self):
+        """Create the 'make' arguments for a toolchain
+
+        This is only used when the toolchain is being overridden. Since the
+        U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
+        environment (and MakeEnvironment()) to override these values. This
+        function returns the arguments to accomplish this.
+
+        Returns:
+            List of arguments to pass to 'make'
+        """
+        if self.override_toolchain:
+            return ['HOSTCC=%s' % self.override_toolchain,
+                    'CC=%s' % self.override_toolchain]
+        return []
+

 
 class Toolchains:
     """Manage a list of toolchains for building U-Boot
 
 class Toolchains:
     """Manage a list of toolchains for building U-Boot


@@ -181,10 +235,11 @@ class Toolchains:
         paths: List of paths to check for toolchains (may contain wildcards)
     """
 
         paths: List of paths to check for toolchains (may contain wildcards)
     """
 

-    def __init__(self):
+    def __init__(self, override_toolchain=None):

         self.toolchains = {}
         self.prefixes = {}
         self.paths = []
         self.toolchains = {}
         self.prefixes = {}
         self.paths = []

+        self.override_toolchain = override_toolchain

         self._make_flags = dict(bsettings.GetItems('make-flags'))
 
     def GetPathList(self, show_warning=True):
         self._make_flags = dict(bsettings.GetItems('make-flags'))
 
     def GetPathList(self, show_warning=True):


@@ -199,11 +254,11 @@ class Toolchains:
         """
         toolchains = bsettings.GetItems('toolchain')
         if show_warning and not toolchains:
         """
         toolchains = bsettings.GetItems('toolchain')
         if show_warning and not toolchains:

-            print ("Warning: No tool chains. Please run 'buildman "
+            print(("Warning: No tool chains. Please run 'buildman "

                    "--fetch-arch all' to download all available toolchains, or "
                    "add a [toolchain] section to your buildman config file "
                    "%s. See README for details" %
                    "--fetch-arch all' to download all available toolchains, or "
                    "add a [toolchain] section to your buildman config file "
                    "%s. See README for details" %

-                   bsettings.config_fname)
+                   bsettings.config_fname))

 
         paths = []
         for name, value in toolchains:
 
         paths = []
         for name, value in toolchains:


@@ -235,7 +290,8 @@ class Toolchains:
             priority: Priority to use for this toolchain
             arch: Toolchain architecture, or None if not known
         """
             priority: Priority to use for this toolchain
             arch: Toolchain architecture, or None if not known
         """

-        toolchain = Toolchain(fname, test, verbose, priority, arch)
+        toolchain = Toolchain(fname, test, verbose, priority, arch,
+                              self.override_toolchain)

         add_it = toolchain.ok
         if toolchain.arch in self.toolchains:
             add_it = (toolchain.priority <
         add_it = toolchain.ok
         if toolchain.arch in self.toolchains:
             add_it = (toolchain.priority <


@@ -243,10 +299,10 @@ class Toolchains:
         if add_it:
             self.toolchains[toolchain.arch] = toolchain
         elif verbose:
         if add_it:
             self.toolchains[toolchain.arch] = toolchain
         elif verbose:

-            print ("Toolchain '%s' at priority %d will be ignored because "
+            print(("Toolchain '%s' at priority %d will be ignored because "

                    "another toolchain for arch '%s' has priority %d" %
                    (toolchain.gcc, toolchain.priority, toolchain.arch,
                    "another toolchain for arch '%s' has priority %d" %
                    (toolchain.gcc, toolchain.priority, toolchain.arch,

-                    self.toolchains[toolchain.arch].priority))
+                    self.toolchains[toolchain.arch].priority)))

 
     def ScanPath(self, path, verbose):
         """Scan a path for a valid toolchain
 
     def ScanPath(self, path, verbose):
         """Scan a path for a valid toolchain


@@ -260,9 +316,9 @@ class Toolchains:
         fnames = []
         for subdir in ['.', 'bin', 'usr/bin']:
             dirname = os.path.join(path, subdir)
         fnames = []
         for subdir in ['.', 'bin', 'usr/bin']:
             dirname = os.path.join(path, subdir)

-            if verbose: print "      - looking in '%s'" % dirname
+            if verbose: print("      - looking in '%s'" % dirname)

             for fname in glob.glob(dirname + '/*gcc'):
             for fname in glob.glob(dirname + '/*gcc'):

-                if verbose: print "         - found '%s'" % fname
+                if verbose: print("         - found '%s'" % fname)

                 fnames.append(fname)
         return fnames
 
                 fnames.append(fname)
         return fnames
 


@@ -292,9 +348,9 @@ class Toolchains:
         Args:
             verbose: True to print out progress information
         """
         Args:
             verbose: True to print out progress information
         """

-        if verbose: print 'Scanning for tool chains'
+        if verbose: print('Scanning for tool chains')

         for name, value in self.prefixes:
         for name, value in self.prefixes:

-            if verbose: print "   - scanning prefix '%s'" % value
+            if verbose: print("   - scanning prefix '%s'" % value)

             if os.path.exists(value):
                 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
                 continue
             if os.path.exists(value):
                 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
                 continue


@@ -306,10 +362,10 @@ class Toolchains:
             for f in fname_list:
                 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
             if not fname_list:
             for f in fname_list:
                 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
             if not fname_list:

-                raise ValueError, ("No tool chain found for prefix '%s'" %
+                raise ValueError("No tool chain found for prefix '%s'" %

                                    value)
         for path in self.paths:
                                    value)
         for path in self.paths:

-            if verbose: print "   - scanning path '%s'" % path
+            if verbose: print("   - scanning path '%s'" % path)

             fnames = self.ScanPath(path, verbose)
             for fname in fnames:
                 self.Add(fname, True, verbose)
             fnames = self.ScanPath(path, verbose)
             for fname in fnames:
                 self.Add(fname, True, verbose)


@@ -317,13 +373,13 @@ class Toolchains:
     def List(self):
         """List out the selected toolchains for each architecture"""
         col = terminal.Color()
     def List(self):
         """List out the selected toolchains for each architecture"""
         col = terminal.Color()

-        print col.Color(col.BLUE, 'List of available toolchains (%d):' %
-                        len(self.toolchains))
+        print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
+                        len(self.toolchains)))

         if len(self.toolchains):
         if len(self.toolchains):

-            for key, value in sorted(self.toolchains.iteritems()):
-                print '%-10s: %s' % (key, value.gcc)
+            for key, value in sorted(self.toolchains.items()):
+                print('%-10s: %s' % (key, value.gcc))

         else:
         else:

-            print 'None'
+            print('None')

 
     def Select(self, arch):
         """Returns the toolchain for a given architecture
 
     def Select(self, arch):
         """Returns the toolchain for a given architecture


@@ -341,7 +397,7 @@ class Toolchains:
                         return self.toolchains[alias]
 
         if not arch in self.toolchains:
                         return self.toolchains[alias]
 
         if not arch in self.toolchains:

-            raise ValueError, ("No tool chain found for arch '%s'" % arch)
+            raise ValueError("No tool chain found for arch '%s'" % arch)

         return self.toolchains[arch]
 
     def ResolveReferences(self, var_dict, args):
         return self.toolchains[arch]
 
     def ResolveReferences(self, var_dict, args):


@@ -405,9 +461,10 @@ class Toolchains:
         self._make_flags['target'] = board.target
         arg_str = self.ResolveReferences(self._make_flags,
                            self._make_flags.get(board.target, ''))
         self._make_flags['target'] = board.target
         arg_str = self.ResolveReferences(self._make_flags,
                            self._make_flags.get(board.target, ''))

-        args = arg_str.split(' ')
+        args = re.findall("(?:\".*?\"|\S)+", arg_str)

         i = 0
         while i < len(args):
         i = 0
         while i < len(args):

+            args[i] = args[i].replace('"', '')

             if not args[i]:
                 del args[i]
             else:
             if not args[i]:
                 del args[i]
             else:


@@ -430,14 +487,16 @@ class Toolchains:
                 URL containing this toolchain, if avaialble, else None
         """
         arch = command.OutputOneLine('uname', '-m')
                 URL containing this toolchain, if avaialble, else None
         """
         arch = command.OutputOneLine('uname', '-m')

+        if arch == 'aarch64':
+            arch = 'arm64'

         base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
         base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'

-        versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
+        versions = ['9.2.0', '7.3.0', '6.4.0', '4.9.4']

         links = []
         for version in versions:
             url = '%s/%s/%s/' % (base, arch, version)
         links = []
         for version in versions:
             url = '%s/%s/%s/' % (base, arch, version)

-            print 'Checking: %s' % url
-            response = urllib2.urlopen(url)
-            html = response.read()
+            print('Checking: %s' % url)
+            response = urllib.request.urlopen(url)
+            html = tools.ToString(response.read())

             parser = MyHTMLParser(fetch_arch)
             parser.feed(html)
             if fetch_arch == 'list':
             parser = MyHTMLParser(fetch_arch)
             parser.feed(html)
             if fetch_arch == 'list':


@@ -459,14 +518,14 @@ class Toolchains:
                 Full path to the downloaded archive file in that directory,
                     or None if there was an error while downloading
         """
                 Full path to the downloaded archive file in that directory,
                     or None if there was an error while downloading
         """

-        print 'Downloading: %s' % url
+        print('Downloading: %s' % url)

         leaf = url.split('/')[-1]
         tmpdir = tempfile.mkdtemp('.buildman')
         leaf = url.split('/')[-1]
         tmpdir = tempfile.mkdtemp('.buildman')

-        response = urllib2.urlopen(url)
+        response = urllib.request.urlopen(url)

         fname = os.path.join(tmpdir, leaf)
         fd = open(fname, 'wb')
         meta = response.info()
         fname = os.path.join(tmpdir, leaf)
         fd = open(fname, 'wb')
         meta = response.info()

-        size = int(meta.getheaders('Content-Length')[0])
+        size = int(meta.get('Content-Length'))

         done = 0
         block_size = 1 << 16
         status = ''
         done = 0
         block_size = 1 << 16
         status = ''


@@ -475,19 +534,19 @@ class Toolchains:
         while True:
             buffer = response.read(block_size)
             if not buffer:
         while True:
             buffer = response.read(block_size)
             if not buffer:

-                print chr(8) * (len(status) + 1), '\r',
+                print(chr(8) * (len(status) + 1), '\r', end=' ')

                 break
 
             done += len(buffer)
             fd.write(buffer)
                 break
 
             done += len(buffer)
             fd.write(buffer)

-            status = r'%10d MiB  [%3d%%]' % (done / 1024 / 1024,
-                                             done * 100 / size)
+            status = r'%10d MiB  [%3d%%]' % (done // 1024 // 1024,
+                                             done * 100 // size)

             status = status + chr(8) * (len(status) + 1)
             status = status + chr(8) * (len(status) + 1)

-            print status,
+            print(status, end=' ')

             sys.stdout.flush()
         fd.close()
         if done != size:
             sys.stdout.flush()
         fd.close()
         if done != size:

-            print 'Error, failed to download'
+            print('Error, failed to download')

             os.remove(fname)
             fname = None
         return tmpdir, fname
             os.remove(fname)
             fname = None
         return tmpdir, fname


@@ -503,7 +562,8 @@ class Toolchains:
             trailing /
         """
         stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
             trailing /
         """
         stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)

-        return stdout.splitlines()[0][:-1]
+        dirs = stdout.splitlines()[1].split('/')[:2]
+        return '/'.join(dirs)

 
     def TestSettingsHasPath(self, path):
         """Check if buildman will find this toolchain
 
     def TestSettingsHasPath(self, path):
         """Check if buildman will find this toolchain


@@ -517,13 +577,14 @@ class Toolchains:
     def ListArchs(self):
         """List architectures with available toolchains to download"""
         host_arch, archives = self.LocateArchUrl('list')
     def ListArchs(self):
         """List architectures with available toolchains to download"""
         host_arch, archives = self.LocateArchUrl('list')

-        re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
+        re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')

         arch_set = set()
         for archive in archives:
             # Remove the host architecture from the start
             arch = re_arch.match(archive[len(host_arch):])
             if arch:
         arch_set = set()
         for archive in archives:
             # Remove the host architecture from the start
             arch = re_arch.match(archive[len(host_arch):])
             if arch:

-                arch_set.add(arch.group(1))
+                if arch.group(1) != '2.0' and arch.group(1) != '64':
+                    arch_set.add(arch.group(1))

         return sorted(arch_set)
 
     def FetchAndInstall(self, arch):
         return sorted(arch_set)
 
     def FetchAndInstall(self, arch):


@@ -534,11 +595,11 @@ class Toolchains:
         """
         # Fist get the URL for this architecture
         col = terminal.Color()
         """
         # Fist get the URL for this architecture
         col = terminal.Color()

-        print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
+        print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))

         url = self.LocateArchUrl(arch)
         if not url:
         url = self.LocateArchUrl(arch)
         if not url:

-            print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
-                   arch)
+            print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
+                   arch))

             return 2
         home = os.environ['HOME']
         dest = os.path.join(home, '.buildman-toolchains')
             return 2
         home = os.environ['HOME']
         dest = os.path.join(home, '.buildman-toolchains')


@@ -549,28 +610,28 @@ class Toolchains:
         tmpdir, tarfile = self.Download(url)
         if not tarfile:
             return 1
         tmpdir, tarfile = self.Download(url)
         if not tarfile:
             return 1

-        print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
+        print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')

         sys.stdout.flush()
         path = self.Unpack(tarfile, dest)
         os.remove(tarfile)
         os.rmdir(tmpdir)
         sys.stdout.flush()
         path = self.Unpack(tarfile, dest)
         os.remove(tarfile)
         os.rmdir(tmpdir)

-        print
+        print()

 
         # Check that the toolchain works
 
         # Check that the toolchain works

-        print col.Color(col.GREEN, 'Testing')
+        print(col.Color(col.GREEN, 'Testing'))

         dirpath = os.path.join(dest, path)
         compiler_fname_list = self.ScanPath(dirpath, True)
         if not compiler_fname_list:
         dirpath = os.path.join(dest, path)
         compiler_fname_list = self.ScanPath(dirpath, True)
         if not compiler_fname_list:

-            print 'Could not locate C compiler - fetch failed.'
+            print('Could not locate C compiler - fetch failed.')

             return 1
         if len(compiler_fname_list) != 1:
             return 1
         if len(compiler_fname_list) != 1:

-            print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
-                            ', '.join(compiler_fname_list))
+            print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
+                            ', '.join(compiler_fname_list)))

         toolchain = Toolchain(compiler_fname_list[0], True, True)
 
         # Make sure that it will be found by buildman
         if not self.TestSettingsHasPath(dirpath):
         toolchain = Toolchain(compiler_fname_list[0], True, True)
 
         # Make sure that it will be found by buildman
         if not self.TestSettingsHasPath(dirpath):

-            print ("Adding 'download' to config file '%s'" %
-                   bsettings.config_fname)
+            print(("Adding 'download' to config file '%s'" %
+                   bsettings.config_fname))

             bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
         return 0
             bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
         return 0