1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
19 # Simple class to collect links from a page
20 class MyHTMLParser(HTMLParser):
21 def __init__(self, arch):
22 """Create a new parser
24 After the parser runs, self.links will be set to a list of the links
25 to .xz archives found in the page, and self.arch_link will be set to
26 the one for the given architecture (or None if not found).
29 arch: Architecture to search for
31 HTMLParser.__init__(self)
34 self._match = '_%s-' % arch
36 def handle_starttag(self, tag, attrs):
38 for tag, value in attrs:
40 if value and value.endswith('.xz'):
41 self.links.append(value)
42 if self._match in value:
43 self.arch_link = value
50 gcc: Full path to C compiler
51 path: Directory path containing C compiler
52 cross: Cross compile string, e.g. 'arm-linux-'
53 arch: Architecture of toolchain as determined from the first
54 component of the filename. E.g. arm-linux-gcc becomes arm
55 priority: Toolchain priority (0=highest, 20=lowest)
57 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC):
58 """Create a new toolchain object.
61 fname: Filename of the gcc component
62 test: True to run the toolchain to test it
63 verbose: True to print out the information
64 priority: Priority to use for this toolchain, or PRIORITY_CALC to
68 self.path = os.path.dirname(fname)
70 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
71 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
72 basename = os.path.basename(fname)
73 pos = basename.rfind('-')
74 self.cross = basename[:pos + 1] if pos != -1 else ''
76 # The architecture is the first part of the name
77 pos = self.cross.find('-')
78 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
80 env = self.MakeEnvironment(False)
82 # As a basic sanity check, run the C compiler with --version
83 cmd = [fname, '--version']
84 if priority == PRIORITY_CALC:
85 self.priority = self.GetPriority(fname)
87 self.priority = priority
89 result = command.RunPipe([cmd], capture=True, env=env,
91 self.ok = result.return_code == 0
93 print 'Tool chain test: ',
95 print 'OK, priority %d' % self.priority
98 print 'Command: ', cmd
104 def GetPriority(self, fname):
105 """Return the priority of the toolchain.
107 Toolchains are ranked according to their suitability by their
111 fname: Filename of toolchain
113 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
115 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
116 '-none-linux-gnueabi', '-uclinux', '-none-eabi',
117 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
118 for prio in range(len(priority_list)):
119 if priority_list[prio] in fname:
120 return PRIORITY_CALC + prio
121 return PRIORITY_CALC + prio
123 def MakeEnvironment(self, full_path):
124 """Returns an environment for using the toolchain.
126 Thie takes the current environment and adds CROSS_COMPILE so that
127 the tool chain will operate correctly.
130 full_path: Return the full path in CROSS_COMPILE and don't set
133 env = dict(os.environ)
135 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
137 env['CROSS_COMPILE'] = self.cross
138 env['PATH'] = self.path + ':' + env['PATH']
144 """Manage a list of toolchains for building U-Boot
146 We select one toolchain for each architecture type
149 toolchains: Dict of Toolchain objects, keyed by architecture name
150 paths: List of paths to check for toolchains (may contain wildcards)
156 self._make_flags = dict(bsettings.GetItems('make-flags'))
158 def GetPathList(self):
159 """Get a list of available toolchain paths
162 List of strings, each a path to a toolchain mentioned in the
163 [toolchain] section of the settings file.
165 toolchains = bsettings.GetItems('toolchain')
167 print ('Warning: No tool chains - please add a [toolchain] section'
168 ' to your buildman config file %s. See README for details' %
169 bsettings.config_fname)
172 for name, value in toolchains:
174 paths += glob.glob(value)
179 def GetSettings(self):
180 self.paths += self.GetPathList()
182 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC):
183 """Add a toolchain to our list
185 We select the given toolchain as our preferred one for its
186 architecture if it is a higher priority than the others.
189 fname: Filename of toolchain's gcc driver
190 test: True to run the toolchain to test it
191 priority: Priority to use for this toolchain
193 toolchain = Toolchain(fname, test, verbose, priority)
194 add_it = toolchain.ok
195 if toolchain.arch in self.toolchains:
196 add_it = (toolchain.priority <
197 self.toolchains[toolchain.arch].priority)
199 self.toolchains[toolchain.arch] = toolchain
201 print ("Toolchain '%s' at priority %d will be ignored because "
202 "another toolchain for arch '%s' has priority %d" %
203 (toolchain.gcc, toolchain.priority, toolchain.arch,
204 self.toolchains[toolchain.arch].priority))
206 def ScanPath(self, path, verbose):
207 """Scan a path for a valid toolchain
211 verbose: True to print out progress information
213 Filename of C compiler if found, else None
216 for subdir in ['.', 'bin', 'usr/bin']:
217 dirname = os.path.join(path, subdir)
218 if verbose: print " - looking in '%s'" % dirname
219 for fname in glob.glob(dirname + '/*gcc'):
220 if verbose: print " - found '%s'" % fname
225 def Scan(self, verbose):
226 """Scan for available toolchains and select the best for each arch.
228 We look for all the toolchains we can file, figure out the
229 architecture for each, and whether it works. Then we select the
230 highest priority toolchain for each arch.
233 verbose: True to print out progress information
235 if verbose: print 'Scanning for tool chains'
236 for path in self.paths:
237 if verbose: print " - scanning path '%s'" % path
238 fnames = self.ScanPath(path, verbose)
240 self.Add(fname, True, verbose)
243 """List out the selected toolchains for each architecture"""
244 print 'List of available toolchains (%d):' % len(self.toolchains)
245 if len(self.toolchains):
246 for key, value in sorted(self.toolchains.iteritems()):
247 print '%-10s: %s' % (key, value.gcc)
251 def Select(self, arch):
252 """Returns the toolchain for a given architecture
255 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
258 toolchain object, or None if none found
260 for tag, value in bsettings.GetItems('toolchain-alias'):
262 for alias in value.split():
263 if alias in self.toolchains:
264 return self.toolchains[alias]
266 if not arch in self.toolchains:
267 raise ValueError, ("No tool chain found for arch '%s'" % arch)
268 return self.toolchains[arch]
270 def ResolveReferences(self, var_dict, args):
271 """Resolve variable references in a string
273 This converts ${blah} within the string to the value of blah.
274 This function works recursively.
277 var_dict: Dictionary containing variables and their values
278 args: String containing make arguments
282 >>> bsettings.Setup()
283 >>> tcs = Toolchains()
284 >>> tcs.Add('fred', False)
285 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
287 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
289 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
290 'this=OBLIQUE_setfi2ndrstnd'
292 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
295 m = re_var.search(args)
298 lookup = m.group(0)[2:-1]
299 value = var_dict.get(lookup, '')
300 args = args[:m.start(0)] + value + args[m.end(0):]
303 def GetMakeArguments(self, board):
304 """Returns 'make' arguments for a given board
306 The flags are in a section called 'make-flags'. Flags are named
307 after the target they represent, for example snapper9260=TESTING=1
308 will pass TESTING=1 to make when building the snapper9260 board.
310 References to other boards can be added in the string also. For
314 at91-boards=ENABLE_AT91_TEST=1
315 snapper9260=${at91-boards} BUILD_TAG=442
316 snapper9g45=${at91-boards} BUILD_TAG=443
318 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
319 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
321 A special 'target' variable is set to the board target.
324 board: Board object for the board to check.
326 'make' flags for that board, or '' if none
328 self._make_flags['target'] = board.target
329 arg_str = self.ResolveReferences(self._make_flags,
330 self._make_flags.get(board.target, ''))
331 args = arg_str.split(' ')
340 def LocateArchUrl(self, fetch_arch):
341 """Find a toolchain available online
343 Look in standard places for available toolchains. At present the
344 only standard place is at kernel.org.
347 arch: Architecture to look for, or 'list' for all
349 If fetch_arch is 'list', a tuple:
350 Machine architecture (e.g. x86_64)
353 URL containing this toolchain, if avaialble, else None
355 arch = command.OutputOneLine('uname', '-m')
356 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
357 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
359 for version in versions:
360 url = '%s/%s/%s/' % (base, arch, version)
361 print 'Checking: %s' % url
362 response = urllib2.urlopen(url)
363 html = response.read()
364 parser = MyHTMLParser(fetch_arch)
366 if fetch_arch == 'list':
367 links += parser.links
368 elif parser.arch_link:
369 return url + parser.arch_link
370 if fetch_arch == 'list':
374 def Download(self, url):
375 """Download a file to a temporary directory
381 Temporary directory name
382 Full path to the downloaded archive file in that directory,
383 or None if there was an error while downloading
385 print 'Downloading: %s' % url
386 leaf = url.split('/')[-1]
387 tmpdir = tempfile.mkdtemp('.buildman')
388 response = urllib2.urlopen(url)
389 fname = os.path.join(tmpdir, leaf)
390 fd = open(fname, 'wb')
391 meta = response.info()
392 size = int(meta.getheaders('Content-Length')[0])
397 # Read the file in chunks and show progress as we go
399 buffer = response.read(block_size)
401 print chr(8) * (len(status) + 1), '\r',
406 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
408 status = status + chr(8) * (len(status) + 1)
413 print 'Error, failed to download'
418 def Unpack(self, fname, dest):
422 fname: Filename to unpack
423 dest: Destination directory
425 Directory name of the first entry in the archive, without the
428 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
429 return stdout.splitlines()[0][:-1]
431 def TestSettingsHasPath(self, path):
432 """Check if builmand will find this toolchain
435 True if the path is in settings, False if not
437 paths = self.GetPathList()
441 """List architectures with available toolchains to download"""
442 host_arch, archives = self.LocateArchUrl('list')
443 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
445 for archive in archives:
446 # Remove the host architecture from the start
447 arch = re_arch.match(archive[len(host_arch):])
449 arch_set.add(arch.group(1))
450 return sorted(arch_set)
452 def FetchAndInstall(self, arch):
453 """Fetch and install a new toolchain
456 Architecture to fetch, or 'list' to list
458 # Fist get the URL for this architecture
459 url = self.LocateArchUrl(arch)
461 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
464 home = os.environ['HOME']
465 dest = os.path.join(home, '.buildman-toolchains')
466 if not os.path.exists(dest):
469 # Download the tar file for this toolchain and unpack it
470 tmpdir, tarfile = self.Download(url)
473 print 'Unpacking to: %s' % dest,
475 path = self.Unpack(tarfile, dest)
480 # Check that the toolchain works
482 dirpath = os.path.join(dest, path)
483 compiler_fname_list = self.ScanPath(dirpath, True)
484 if not compiler_fname_list:
485 print 'Could not locate C compiler - fetch failed.'
487 if len(compiler_fname_list) != 1:
488 print ('Internal error, ambiguous toolchains: %s' %
489 (', '.join(compiler_fname)))
491 toolchain = Toolchain(compiler_fname_list[0], True, True)
493 # Make sure that it will be found by buildman
494 if not self.TestSettingsHasPath(dirpath):
495 print ("Adding 'download' to config file '%s'" %
496 bsettings.config_fname)
497 tools_dir = os.path.dirname(dirpath)
498 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)