577 lines
20 KiB
Python
577 lines
20 KiB
Python
# Copyright (c) 2012 The Chromium OS Authors.
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0+
|
|
#
|
|
|
|
import re
|
|
import glob
|
|
from HTMLParser import HTMLParser
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import urllib2
|
|
|
|
import bsettings
|
|
import command
|
|
import terminal
|
|
|
|
(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
|
|
PRIORITY_CALC) = range(4)
|
|
|
|
# Simple class to collect links from a page
|
|
class MyHTMLParser(HTMLParser):
|
|
def __init__(self, arch):
|
|
"""Create a new parser
|
|
|
|
After the parser runs, self.links will be set to a list of the links
|
|
to .xz archives found in the page, and self.arch_link will be set to
|
|
the one for the given architecture (or None if not found).
|
|
|
|
Args:
|
|
arch: Architecture to search for
|
|
"""
|
|
HTMLParser.__init__(self)
|
|
self.arch_link = None
|
|
self.links = []
|
|
self._match = '_%s-' % arch
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag == 'a':
|
|
for tag, value in attrs:
|
|
if tag == 'href':
|
|
if value and value.endswith('.xz'):
|
|
self.links.append(value)
|
|
if self._match in value:
|
|
self.arch_link = value
|
|
|
|
|
|
class Toolchain:
|
|
"""A single toolchain
|
|
|
|
Public members:
|
|
gcc: Full path to C compiler
|
|
path: Directory path containing C compiler
|
|
cross: Cross compile string, e.g. 'arm-linux-'
|
|
arch: Architecture of toolchain as determined from the first
|
|
component of the filename. E.g. arm-linux-gcc becomes arm
|
|
priority: Toolchain priority (0=highest, 20=lowest)
|
|
"""
|
|
def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
|
|
arch=None):
|
|
"""Create a new toolchain object.
|
|
|
|
Args:
|
|
fname: Filename of the gcc component
|
|
test: True to run the toolchain to test it
|
|
verbose: True to print out the information
|
|
priority: Priority to use for this toolchain, or PRIORITY_CALC to
|
|
calculate it
|
|
"""
|
|
self.gcc = fname
|
|
self.path = os.path.dirname(fname)
|
|
|
|
# Find the CROSS_COMPILE prefix to use for U-Boot. For example,
|
|
# 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
|
|
basename = os.path.basename(fname)
|
|
pos = basename.rfind('-')
|
|
self.cross = basename[:pos + 1] if pos != -1 else ''
|
|
|
|
# The architecture is the first part of the name
|
|
pos = self.cross.find('-')
|
|
if arch:
|
|
self.arch = arch
|
|
else:
|
|
self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
|
|
|
|
env = self.MakeEnvironment(False)
|
|
|
|
# As a basic sanity check, run the C compiler with --version
|
|
cmd = [fname, '--version']
|
|
if priority == PRIORITY_CALC:
|
|
self.priority = self.GetPriority(fname)
|
|
else:
|
|
self.priority = priority
|
|
if test:
|
|
result = command.RunPipe([cmd], capture=True, env=env,
|
|
raise_on_error=False)
|
|
self.ok = result.return_code == 0
|
|
if verbose:
|
|
print 'Tool chain test: ',
|
|
if self.ok:
|
|
print "OK, arch='%s', priority %d" % (self.arch,
|
|
self.priority)
|
|
else:
|
|
print 'BAD'
|
|
print 'Command: ', cmd
|
|
print result.stdout
|
|
print result.stderr
|
|
else:
|
|
self.ok = True
|
|
|
|
def GetPriority(self, fname):
|
|
"""Return the priority of the toolchain.
|
|
|
|
Toolchains are ranked according to their suitability by their
|
|
filename prefix.
|
|
|
|
Args:
|
|
fname: Filename of toolchain
|
|
Returns:
|
|
Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
|
|
"""
|
|
priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
|
|
'-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
|
|
'-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
|
|
'-linux-gnueabihf', '-le-linux', '-uclinux']
|
|
for prio in range(len(priority_list)):
|
|
if priority_list[prio] in fname:
|
|
return PRIORITY_CALC + prio
|
|
return PRIORITY_CALC + prio
|
|
|
|
def GetWrapper(self, show_warning=True):
|
|
"""Get toolchain wrapper from the setting file.
|
|
"""
|
|
value = ''
|
|
for name, value in bsettings.GetItems('toolchain-wrapper'):
|
|
if not value:
|
|
print "Warning: Wrapper not found"
|
|
if value:
|
|
value = value + ' '
|
|
|
|
return value
|
|
|
|
def MakeEnvironment(self, full_path):
|
|
"""Returns an environment for using the toolchain.
|
|
|
|
Thie takes the current environment and adds CROSS_COMPILE so that
|
|
the tool chain will operate correctly. This also disables localized
|
|
output and possibly unicode encoded output of all build tools by
|
|
adding LC_ALL=C.
|
|
|
|
Args:
|
|
full_path: Return the full path in CROSS_COMPILE and don't set
|
|
PATH
|
|
"""
|
|
env = dict(os.environ)
|
|
wrapper = self.GetWrapper()
|
|
|
|
if full_path:
|
|
env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
|
|
else:
|
|
env['CROSS_COMPILE'] = wrapper + self.cross
|
|
env['PATH'] = self.path + ':' + env['PATH']
|
|
|
|
env['LC_ALL'] = 'C'
|
|
|
|
return env
|
|
|
|
|
|
class Toolchains:
|
|
"""Manage a list of toolchains for building U-Boot
|
|
|
|
We select one toolchain for each architecture type
|
|
|
|
Public members:
|
|
toolchains: Dict of Toolchain objects, keyed by architecture name
|
|
prefixes: Dict of prefixes to check, keyed by architecture. This can
|
|
be a full path and toolchain prefix, for example
|
|
{'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
|
|
something on the search path, for example
|
|
{'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
|
|
paths: List of paths to check for toolchains (may contain wildcards)
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.toolchains = {}
|
|
self.prefixes = {}
|
|
self.paths = []
|
|
self._make_flags = dict(bsettings.GetItems('make-flags'))
|
|
|
|
def GetPathList(self, show_warning=True):
|
|
"""Get a list of available toolchain paths
|
|
|
|
Args:
|
|
show_warning: True to show a warning if there are no tool chains.
|
|
|
|
Returns:
|
|
List of strings, each a path to a toolchain mentioned in the
|
|
[toolchain] section of the settings file.
|
|
"""
|
|
toolchains = bsettings.GetItems('toolchain')
|
|
if show_warning and not toolchains:
|
|
print ("Warning: No tool chains. Please run 'buildman "
|
|
"--fetch-arch all' to download all available toolchains, or "
|
|
"add a [toolchain] section to your buildman config file "
|
|
"%s. See README for details" %
|
|
bsettings.config_fname)
|
|
|
|
paths = []
|
|
for name, value in toolchains:
|
|
if '*' in value:
|
|
paths += glob.glob(value)
|
|
else:
|
|
paths.append(value)
|
|
return paths
|
|
|
|
def GetSettings(self, show_warning=True):
|
|
"""Get toolchain settings from the settings file.
|
|
|
|
Args:
|
|
show_warning: True to show a warning if there are no tool chains.
|
|
"""
|
|
self.prefixes = bsettings.GetItems('toolchain-prefix')
|
|
self.paths += self.GetPathList(show_warning)
|
|
|
|
def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
|
|
arch=None):
|
|
"""Add a toolchain to our list
|
|
|
|
We select the given toolchain as our preferred one for its
|
|
architecture if it is a higher priority than the others.
|
|
|
|
Args:
|
|
fname: Filename of toolchain's gcc driver
|
|
test: True to run the toolchain to test it
|
|
priority: Priority to use for this toolchain
|
|
arch: Toolchain architecture, or None if not known
|
|
"""
|
|
toolchain = Toolchain(fname, test, verbose, priority, arch)
|
|
add_it = toolchain.ok
|
|
if toolchain.arch in self.toolchains:
|
|
add_it = (toolchain.priority <
|
|
self.toolchains[toolchain.arch].priority)
|
|
if add_it:
|
|
self.toolchains[toolchain.arch] = toolchain
|
|
elif verbose:
|
|
print ("Toolchain '%s' at priority %d will be ignored because "
|
|
"another toolchain for arch '%s' has priority %d" %
|
|
(toolchain.gcc, toolchain.priority, toolchain.arch,
|
|
self.toolchains[toolchain.arch].priority))
|
|
|
|
def ScanPath(self, path, verbose):
|
|
"""Scan a path for a valid toolchain
|
|
|
|
Args:
|
|
path: Path to scan
|
|
verbose: True to print out progress information
|
|
Returns:
|
|
Filename of C compiler if found, else None
|
|
"""
|
|
fnames = []
|
|
for subdir in ['.', 'bin', 'usr/bin']:
|
|
dirname = os.path.join(path, subdir)
|
|
if verbose: print " - looking in '%s'" % dirname
|
|
for fname in glob.glob(dirname + '/*gcc'):
|
|
if verbose: print " - found '%s'" % fname
|
|
fnames.append(fname)
|
|
return fnames
|
|
|
|
def ScanPathEnv(self, fname):
|
|
"""Scan the PATH environment variable for a given filename.
|
|
|
|
Args:
|
|
fname: Filename to scan for
|
|
Returns:
|
|
List of matching pathanames, or [] if none
|
|
"""
|
|
pathname_list = []
|
|
for path in os.environ["PATH"].split(os.pathsep):
|
|
path = path.strip('"')
|
|
pathname = os.path.join(path, fname)
|
|
if os.path.exists(pathname):
|
|
pathname_list.append(pathname)
|
|
return pathname_list
|
|
|
|
def Scan(self, verbose):
|
|
"""Scan for available toolchains and select the best for each arch.
|
|
|
|
We look for all the toolchains we can file, figure out the
|
|
architecture for each, and whether it works. Then we select the
|
|
highest priority toolchain for each arch.
|
|
|
|
Args:
|
|
verbose: True to print out progress information
|
|
"""
|
|
if verbose: print 'Scanning for tool chains'
|
|
for name, value in self.prefixes:
|
|
if verbose: print " - scanning prefix '%s'" % value
|
|
if os.path.exists(value):
|
|
self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
|
|
continue
|
|
fname = value + 'gcc'
|
|
if os.path.exists(fname):
|
|
self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
|
|
continue
|
|
fname_list = self.ScanPathEnv(fname)
|
|
for f in fname_list:
|
|
self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
|
|
if not fname_list:
|
|
raise ValueError, ("No tool chain found for prefix '%s'" %
|
|
value)
|
|
for path in self.paths:
|
|
if verbose: print " - scanning path '%s'" % path
|
|
fnames = self.ScanPath(path, verbose)
|
|
for fname in fnames:
|
|
self.Add(fname, True, verbose)
|
|
|
|
def List(self):
|
|
"""List out the selected toolchains for each architecture"""
|
|
col = terminal.Color()
|
|
print col.Color(col.BLUE, 'List of available toolchains (%d):' %
|
|
len(self.toolchains))
|
|
if len(self.toolchains):
|
|
for key, value in sorted(self.toolchains.iteritems()):
|
|
print '%-10s: %s' % (key, value.gcc)
|
|
else:
|
|
print 'None'
|
|
|
|
def Select(self, arch):
|
|
"""Returns the toolchain for a given architecture
|
|
|
|
Args:
|
|
args: Name of architecture (e.g. 'arm', 'ppc_8xx')
|
|
|
|
returns:
|
|
toolchain object, or None if none found
|
|
"""
|
|
for tag, value in bsettings.GetItems('toolchain-alias'):
|
|
if arch == tag:
|
|
for alias in value.split():
|
|
if alias in self.toolchains:
|
|
return self.toolchains[alias]
|
|
|
|
if not arch in self.toolchains:
|
|
raise ValueError, ("No tool chain found for arch '%s'" % arch)
|
|
return self.toolchains[arch]
|
|
|
|
def ResolveReferences(self, var_dict, args):
|
|
"""Resolve variable references in a string
|
|
|
|
This converts ${blah} within the string to the value of blah.
|
|
This function works recursively.
|
|
|
|
Args:
|
|
var_dict: Dictionary containing variables and their values
|
|
args: String containing make arguments
|
|
Returns:
|
|
Resolved string
|
|
|
|
>>> bsettings.Setup()
|
|
>>> tcs = Toolchains()
|
|
>>> tcs.Add('fred', False)
|
|
>>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
|
|
'second' : '2nd'}
|
|
>>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
|
|
'this=OBLIQUE_set'
|
|
>>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
|
|
'this=OBLIQUE_setfi2ndrstnd'
|
|
"""
|
|
re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
|
|
|
|
while True:
|
|
m = re_var.search(args)
|
|
if not m:
|
|
break
|
|
lookup = m.group(0)[2:-1]
|
|
value = var_dict.get(lookup, '')
|
|
args = args[:m.start(0)] + value + args[m.end(0):]
|
|
return args
|
|
|
|
def GetMakeArguments(self, board):
|
|
"""Returns 'make' arguments for a given board
|
|
|
|
The flags are in a section called 'make-flags'. Flags are named
|
|
after the target they represent, for example snapper9260=TESTING=1
|
|
will pass TESTING=1 to make when building the snapper9260 board.
|
|
|
|
References to other boards can be added in the string also. For
|
|
example:
|
|
|
|
[make-flags]
|
|
at91-boards=ENABLE_AT91_TEST=1
|
|
snapper9260=${at91-boards} BUILD_TAG=442
|
|
snapper9g45=${at91-boards} BUILD_TAG=443
|
|
|
|
This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
|
|
and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
|
|
|
|
A special 'target' variable is set to the board target.
|
|
|
|
Args:
|
|
board: Board object for the board to check.
|
|
Returns:
|
|
'make' flags for that board, or '' if none
|
|
"""
|
|
self._make_flags['target'] = board.target
|
|
arg_str = self.ResolveReferences(self._make_flags,
|
|
self._make_flags.get(board.target, ''))
|
|
args = arg_str.split(' ')
|
|
i = 0
|
|
while i < len(args):
|
|
if not args[i]:
|
|
del args[i]
|
|
else:
|
|
i += 1
|
|
return args
|
|
|
|
def LocateArchUrl(self, fetch_arch):
|
|
"""Find a toolchain available online
|
|
|
|
Look in standard places for available toolchains. At present the
|
|
only standard place is at kernel.org.
|
|
|
|
Args:
|
|
arch: Architecture to look for, or 'list' for all
|
|
Returns:
|
|
If fetch_arch is 'list', a tuple:
|
|
Machine architecture (e.g. x86_64)
|
|
List of toolchains
|
|
else
|
|
URL containing this toolchain, if avaialble, else None
|
|
"""
|
|
arch = command.OutputOneLine('uname', '-m')
|
|
base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
|
|
versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
|
|
links = []
|
|
for version in versions:
|
|
url = '%s/%s/%s/' % (base, arch, version)
|
|
print 'Checking: %s' % url
|
|
response = urllib2.urlopen(url)
|
|
html = response.read()
|
|
parser = MyHTMLParser(fetch_arch)
|
|
parser.feed(html)
|
|
if fetch_arch == 'list':
|
|
links += parser.links
|
|
elif parser.arch_link:
|
|
return url + parser.arch_link
|
|
if fetch_arch == 'list':
|
|
return arch, links
|
|
return None
|
|
|
|
def Download(self, url):
|
|
"""Download a file to a temporary directory
|
|
|
|
Args:
|
|
url: URL to download
|
|
Returns:
|
|
Tuple:
|
|
Temporary directory name
|
|
Full path to the downloaded archive file in that directory,
|
|
or None if there was an error while downloading
|
|
"""
|
|
print 'Downloading: %s' % url
|
|
leaf = url.split('/')[-1]
|
|
tmpdir = tempfile.mkdtemp('.buildman')
|
|
response = urllib2.urlopen(url)
|
|
fname = os.path.join(tmpdir, leaf)
|
|
fd = open(fname, 'wb')
|
|
meta = response.info()
|
|
size = int(meta.getheaders('Content-Length')[0])
|
|
done = 0
|
|
block_size = 1 << 16
|
|
status = ''
|
|
|
|
# Read the file in chunks and show progress as we go
|
|
while True:
|
|
buffer = response.read(block_size)
|
|
if not buffer:
|
|
print chr(8) * (len(status) + 1), '\r',
|
|
break
|
|
|
|
done += len(buffer)
|
|
fd.write(buffer)
|
|
status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
|
|
done * 100 / size)
|
|
status = status + chr(8) * (len(status) + 1)
|
|
print status,
|
|
sys.stdout.flush()
|
|
fd.close()
|
|
if done != size:
|
|
print 'Error, failed to download'
|
|
os.remove(fname)
|
|
fname = None
|
|
return tmpdir, fname
|
|
|
|
def Unpack(self, fname, dest):
|
|
"""Unpack a tar file
|
|
|
|
Args:
|
|
fname: Filename to unpack
|
|
dest: Destination directory
|
|
Returns:
|
|
Directory name of the first entry in the archive, without the
|
|
trailing /
|
|
"""
|
|
stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
|
|
return stdout.splitlines()[0][:-1]
|
|
|
|
def TestSettingsHasPath(self, path):
|
|
"""Check if buildman will find this toolchain
|
|
|
|
Returns:
|
|
True if the path is in settings, False if not
|
|
"""
|
|
paths = self.GetPathList(False)
|
|
return path in paths
|
|
|
|
def ListArchs(self):
|
|
"""List architectures with available toolchains to download"""
|
|
host_arch, archives = self.LocateArchUrl('list')
|
|
re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
|
|
arch_set = set()
|
|
for archive in archives:
|
|
# Remove the host architecture from the start
|
|
arch = re_arch.match(archive[len(host_arch):])
|
|
if arch:
|
|
arch_set.add(arch.group(1))
|
|
return sorted(arch_set)
|
|
|
|
def FetchAndInstall(self, arch):
|
|
"""Fetch and install a new toolchain
|
|
|
|
arch:
|
|
Architecture to fetch, or 'list' to list
|
|
"""
|
|
# Fist get the URL for this architecture
|
|
col = terminal.Color()
|
|
print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
|
|
url = self.LocateArchUrl(arch)
|
|
if not url:
|
|
print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
|
|
arch)
|
|
return 2
|
|
home = os.environ['HOME']
|
|
dest = os.path.join(home, '.buildman-toolchains')
|
|
if not os.path.exists(dest):
|
|
os.mkdir(dest)
|
|
|
|
# Download the tar file for this toolchain and unpack it
|
|
tmpdir, tarfile = self.Download(url)
|
|
if not tarfile:
|
|
return 1
|
|
print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
|
|
sys.stdout.flush()
|
|
path = self.Unpack(tarfile, dest)
|
|
os.remove(tarfile)
|
|
os.rmdir(tmpdir)
|
|
print
|
|
|
|
# Check that the toolchain works
|
|
print col.Color(col.GREEN, 'Testing')
|
|
dirpath = os.path.join(dest, path)
|
|
compiler_fname_list = self.ScanPath(dirpath, True)
|
|
if not compiler_fname_list:
|
|
print 'Could not locate C compiler - fetch failed.'
|
|
return 1
|
|
if len(compiler_fname_list) != 1:
|
|
print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
|
|
', '.join(compiler_fname_list))
|
|
toolchain = Toolchain(compiler_fname_list[0], True, True)
|
|
|
|
# Make sure that it will be found by buildman
|
|
if not self.TestSettingsHasPath(dirpath):
|
|
print ("Adding 'download' to config file '%s'" %
|
|
bsettings.config_fname)
|
|
bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
|
|
return 0
|