646 lines
		
	
	
		
			23 KiB
		
	
	
	
		
			Python
		
	
	
	
			
		
		
	
	
			646 lines
		
	
	
		
			23 KiB
		
	
	
	
		
			Python
		
	
	
	
# SPDX-License-Identifier: GPL-2.0+
 | 
						|
# Copyright (c) 2012 The Chromium OS Authors.
 | 
						|
#
 | 
						|
 | 
						|
import re
 | 
						|
import glob
 | 
						|
from html.parser import HTMLParser
 | 
						|
import os
 | 
						|
import sys
 | 
						|
import tempfile
 | 
						|
import urllib.request, urllib.error, urllib.parse
 | 
						|
 | 
						|
from buildman import bsettings
 | 
						|
from patman import command
 | 
						|
from patman import terminal
 | 
						|
from patman import tools
 | 
						|
 | 
						|
(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
 | 
						|
    PRIORITY_CALC) = list(range(4))
 | 
						|
 | 
						|
(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
 | 
						|
 | 
						|
# Simple class to collect links from a page
 | 
						|
class MyHTMLParser(HTMLParser):
 | 
						|
    def __init__(self, arch):
 | 
						|
        """Create a new parser
 | 
						|
 | 
						|
        After the parser runs, self.links will be set to a list of the links
 | 
						|
        to .xz archives found in the page, and self.arch_link will be set to
 | 
						|
        the one for the given architecture (or None if not found).
 | 
						|
 | 
						|
        Args:
 | 
						|
            arch: Architecture to search for
 | 
						|
        """
 | 
						|
        HTMLParser.__init__(self)
 | 
						|
        self.arch_link = None
 | 
						|
        self.links = []
 | 
						|
        self.re_arch = re.compile('[-_]%s-' % arch)
 | 
						|
 | 
						|
    def handle_starttag(self, tag, attrs):
 | 
						|
        if tag == 'a':
 | 
						|
            for tag, value in attrs:
 | 
						|
                if tag == 'href':
 | 
						|
                    if value and value.endswith('.xz'):
 | 
						|
                        self.links.append(value)
 | 
						|
                        if self.re_arch.search(value):
 | 
						|
                            self.arch_link = value
 | 
						|
 | 
						|
 | 
						|
class Toolchain:
 | 
						|
    """A single toolchain
 | 
						|
 | 
						|
    Public members:
 | 
						|
        gcc: Full path to C compiler
 | 
						|
        path: Directory path containing C compiler
 | 
						|
        cross: Cross compile string, e.g. 'arm-linux-'
 | 
						|
        arch: Architecture of toolchain as determined from the first
 | 
						|
                component of the filename. E.g. arm-linux-gcc becomes arm
 | 
						|
        priority: Toolchain priority (0=highest, 20=lowest)
 | 
						|
        override_toolchain: Toolchain to use for sandbox, overriding the normal
 | 
						|
                one
 | 
						|
    """
 | 
						|
    def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
 | 
						|
                 arch=None, override_toolchain=None):
 | 
						|
        """Create a new toolchain object.
 | 
						|
 | 
						|
        Args:
 | 
						|
            fname: Filename of the gcc component
 | 
						|
            test: True to run the toolchain to test it
 | 
						|
            verbose: True to print out the information
 | 
						|
            priority: Priority to use for this toolchain, or PRIORITY_CALC to
 | 
						|
                calculate it
 | 
						|
        """
 | 
						|
        self.gcc = fname
 | 
						|
        self.path = os.path.dirname(fname)
 | 
						|
        self.override_toolchain = override_toolchain
 | 
						|
 | 
						|
        # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
 | 
						|
        # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
 | 
						|
        basename = os.path.basename(fname)
 | 
						|
        pos = basename.rfind('-')
 | 
						|
        self.cross = basename[:pos + 1] if pos != -1 else ''
 | 
						|
 | 
						|
        # The architecture is the first part of the name
 | 
						|
        pos = self.cross.find('-')
 | 
						|
        if arch:
 | 
						|
            self.arch = arch
 | 
						|
        else:
 | 
						|
            self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
 | 
						|
        if self.arch == 'sandbox' and override_toolchain:
 | 
						|
            self.gcc = override_toolchain
 | 
						|
 | 
						|
        env = self.MakeEnvironment(False)
 | 
						|
 | 
						|
        # As a basic sanity check, run the C compiler with --version
 | 
						|
        cmd = [fname, '--version']
 | 
						|
        if priority == PRIORITY_CALC:
 | 
						|
            self.priority = self.GetPriority(fname)
 | 
						|
        else:
 | 
						|
            self.priority = priority
 | 
						|
        if test:
 | 
						|
            result = command.RunPipe([cmd], capture=True, env=env,
 | 
						|
                                     raise_on_error=False)
 | 
						|
            self.ok = result.return_code == 0
 | 
						|
            if verbose:
 | 
						|
                print('Tool chain test: ', end=' ')
 | 
						|
                if self.ok:
 | 
						|
                    print("OK, arch='%s', priority %d" % (self.arch,
 | 
						|
                                                          self.priority))
 | 
						|
                else:
 | 
						|
                    print('BAD')
 | 
						|
                    print('Command: ', cmd)
 | 
						|
                    print(result.stdout)
 | 
						|
                    print(result.stderr)
 | 
						|
        else:
 | 
						|
            self.ok = True
 | 
						|
 | 
						|
    def GetPriority(self, fname):
 | 
						|
        """Return the priority of the toolchain.
 | 
						|
 | 
						|
        Toolchains are ranked according to their suitability by their
 | 
						|
        filename prefix.
 | 
						|
 | 
						|
        Args:
 | 
						|
            fname: Filename of toolchain
 | 
						|
        Returns:
 | 
						|
            Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
 | 
						|
        """
 | 
						|
        priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
 | 
						|
            '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
 | 
						|
            '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
 | 
						|
            '-linux-gnueabihf', '-le-linux', '-uclinux']
 | 
						|
        for prio in range(len(priority_list)):
 | 
						|
            if priority_list[prio] in fname:
 | 
						|
                return PRIORITY_CALC + prio
 | 
						|
        return PRIORITY_CALC + prio
 | 
						|
 | 
						|
    def GetWrapper(self, show_warning=True):
 | 
						|
        """Get toolchain wrapper from the setting file.
 | 
						|
        """
 | 
						|
        value = ''
 | 
						|
        for name, value in bsettings.GetItems('toolchain-wrapper'):
 | 
						|
            if not value:
 | 
						|
                print("Warning: Wrapper not found")
 | 
						|
        if value:
 | 
						|
            value = value + ' '
 | 
						|
 | 
						|
        return value
 | 
						|
 | 
						|
    def GetEnvArgs(self, which):
 | 
						|
        """Get an environment variable/args value based on the the toolchain
 | 
						|
 | 
						|
        Args:
 | 
						|
            which: VAR_... value to get
 | 
						|
 | 
						|
        Returns:
 | 
						|
            Value of that environment variable or arguments
 | 
						|
        """
 | 
						|
        wrapper = self.GetWrapper()
 | 
						|
        if which == VAR_CROSS_COMPILE:
 | 
						|
            return wrapper + os.path.join(self.path, self.cross)
 | 
						|
        elif which == VAR_PATH:
 | 
						|
            return self.path
 | 
						|
        elif which == VAR_ARCH:
 | 
						|
            return self.arch
 | 
						|
        elif which == VAR_MAKE_ARGS:
 | 
						|
            args = self.MakeArgs()
 | 
						|
            if args:
 | 
						|
                return ' '.join(args)
 | 
						|
            return ''
 | 
						|
        else:
 | 
						|
            raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
 | 
						|
 | 
						|
    def MakeEnvironment(self, full_path):
 | 
						|
        """Returns an environment for using the toolchain.
 | 
						|
 | 
						|
        Thie takes the current environment and adds CROSS_COMPILE so that
 | 
						|
        the tool chain will operate correctly. This also disables localized
 | 
						|
        output and possibly unicode encoded output of all build tools by
 | 
						|
        adding LC_ALL=C.
 | 
						|
 | 
						|
        Note that os.environb is used to obtain the environment, since in some
 | 
						|
        cases the environment many contain non-ASCII characters and we see
 | 
						|
        errors like:
 | 
						|
 | 
						|
          UnicodeEncodeError: 'utf-8' codec can't encode characters in position
 | 
						|
             569-570: surrogates not allowed
 | 
						|
 | 
						|
        Args:
 | 
						|
            full_path: Return the full path in CROSS_COMPILE and don't set
 | 
						|
                PATH
 | 
						|
        Returns:
 | 
						|
            Dict containing the (bytes) environment to use. This is based on the
 | 
						|
            current environment, with changes as needed to CROSS_COMPILE, PATH
 | 
						|
            and LC_ALL.
 | 
						|
        """
 | 
						|
        env = dict(os.environb)
 | 
						|
        wrapper = self.GetWrapper()
 | 
						|
 | 
						|
        if self.override_toolchain:
 | 
						|
            # We'll use MakeArgs() to provide this
 | 
						|
            pass
 | 
						|
        elif full_path:
 | 
						|
            env[b'CROSS_COMPILE'] = tools.ToBytes(
 | 
						|
                wrapper + os.path.join(self.path, self.cross))
 | 
						|
        else:
 | 
						|
            env[b'CROSS_COMPILE'] = tools.ToBytes(wrapper + self.cross)
 | 
						|
            env[b'PATH'] = tools.ToBytes(self.path) + b':' + env[b'PATH']
 | 
						|
 | 
						|
        env[b'LC_ALL'] = b'C'
 | 
						|
 | 
						|
        return env
 | 
						|
 | 
						|
    def MakeArgs(self):
 | 
						|
        """Create the 'make' arguments for a toolchain
 | 
						|
 | 
						|
        This is only used when the toolchain is being overridden. Since the
 | 
						|
        U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
 | 
						|
        environment (and MakeEnvironment()) to override these values. This
 | 
						|
        function returns the arguments to accomplish this.
 | 
						|
 | 
						|
        Returns:
 | 
						|
            List of arguments to pass to 'make'
 | 
						|
        """
 | 
						|
        if self.override_toolchain:
 | 
						|
            return ['HOSTCC=%s' % self.override_toolchain,
 | 
						|
                    'CC=%s' % self.override_toolchain]
 | 
						|
        return []
 | 
						|
 | 
						|
 | 
						|
class Toolchains:
 | 
						|
    """Manage a list of toolchains for building U-Boot
 | 
						|
 | 
						|
    We select one toolchain for each architecture type
 | 
						|
 | 
						|
    Public members:
 | 
						|
        toolchains: Dict of Toolchain objects, keyed by architecture name
 | 
						|
        prefixes: Dict of prefixes to check, keyed by architecture. This can
 | 
						|
            be a full path and toolchain prefix, for example
 | 
						|
            {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
 | 
						|
            something on the search path, for example
 | 
						|
            {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
 | 
						|
        paths: List of paths to check for toolchains (may contain wildcards)
 | 
						|
    """
 | 
						|
 | 
						|
    def __init__(self, override_toolchain=None):
 | 
						|
        self.toolchains = {}
 | 
						|
        self.prefixes = {}
 | 
						|
        self.paths = []
 | 
						|
        self.override_toolchain = override_toolchain
 | 
						|
        self._make_flags = dict(bsettings.GetItems('make-flags'))
 | 
						|
 | 
						|
    def GetPathList(self, show_warning=True):
 | 
						|
        """Get a list of available toolchain paths
 | 
						|
 | 
						|
        Args:
 | 
						|
            show_warning: True to show a warning if there are no tool chains.
 | 
						|
 | 
						|
        Returns:
 | 
						|
            List of strings, each a path to a toolchain mentioned in the
 | 
						|
            [toolchain] section of the settings file.
 | 
						|
        """
 | 
						|
        toolchains = bsettings.GetItems('toolchain')
 | 
						|
        if show_warning and not toolchains:
 | 
						|
            print(("Warning: No tool chains. Please run 'buildman "
 | 
						|
                   "--fetch-arch all' to download all available toolchains, or "
 | 
						|
                   "add a [toolchain] section to your buildman config file "
 | 
						|
                   "%s. See README for details" %
 | 
						|
                   bsettings.config_fname))
 | 
						|
 | 
						|
        paths = []
 | 
						|
        for name, value in toolchains:
 | 
						|
            if '*' in value:
 | 
						|
                paths += glob.glob(value)
 | 
						|
            else:
 | 
						|
                paths.append(value)
 | 
						|
        return paths
 | 
						|
 | 
						|
    def GetSettings(self, show_warning=True):
 | 
						|
        """Get toolchain settings from the settings file.
 | 
						|
 | 
						|
        Args:
 | 
						|
            show_warning: True to show a warning if there are no tool chains.
 | 
						|
        """
 | 
						|
        self.prefixes = bsettings.GetItems('toolchain-prefix')
 | 
						|
        self.paths += self.GetPathList(show_warning)
 | 
						|
 | 
						|
    def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
 | 
						|
            arch=None):
 | 
						|
        """Add a toolchain to our list
 | 
						|
 | 
						|
        We select the given toolchain as our preferred one for its
 | 
						|
        architecture if it is a higher priority than the others.
 | 
						|
 | 
						|
        Args:
 | 
						|
            fname: Filename of toolchain's gcc driver
 | 
						|
            test: True to run the toolchain to test it
 | 
						|
            priority: Priority to use for this toolchain
 | 
						|
            arch: Toolchain architecture, or None if not known
 | 
						|
        """
 | 
						|
        toolchain = Toolchain(fname, test, verbose, priority, arch,
 | 
						|
                              self.override_toolchain)
 | 
						|
        add_it = toolchain.ok
 | 
						|
        if toolchain.arch in self.toolchains:
 | 
						|
            add_it = (toolchain.priority <
 | 
						|
                        self.toolchains[toolchain.arch].priority)
 | 
						|
        if add_it:
 | 
						|
            self.toolchains[toolchain.arch] = toolchain
 | 
						|
        elif verbose:
 | 
						|
            print(("Toolchain '%s' at priority %d will be ignored because "
 | 
						|
                   "another toolchain for arch '%s' has priority %d" %
 | 
						|
                   (toolchain.gcc, toolchain.priority, toolchain.arch,
 | 
						|
                    self.toolchains[toolchain.arch].priority)))
 | 
						|
 | 
						|
    def ScanPath(self, path, verbose):
 | 
						|
        """Scan a path for a valid toolchain
 | 
						|
 | 
						|
        Args:
 | 
						|
            path: Path to scan
 | 
						|
            verbose: True to print out progress information
 | 
						|
        Returns:
 | 
						|
            Filename of C compiler if found, else None
 | 
						|
        """
 | 
						|
        fnames = []
 | 
						|
        for subdir in ['.', 'bin', 'usr/bin']:
 | 
						|
            dirname = os.path.join(path, subdir)
 | 
						|
            if verbose: print("      - looking in '%s'" % dirname)
 | 
						|
            for fname in glob.glob(dirname + '/*gcc'):
 | 
						|
                if verbose: print("         - found '%s'" % fname)
 | 
						|
                fnames.append(fname)
 | 
						|
        return fnames
 | 
						|
 | 
						|
    def ScanPathEnv(self, fname):
 | 
						|
        """Scan the PATH environment variable for a given filename.
 | 
						|
 | 
						|
        Args:
 | 
						|
            fname: Filename to scan for
 | 
						|
        Returns:
 | 
						|
            List of matching pathanames, or [] if none
 | 
						|
        """
 | 
						|
        pathname_list = []
 | 
						|
        for path in os.environ["PATH"].split(os.pathsep):
 | 
						|
            path = path.strip('"')
 | 
						|
            pathname = os.path.join(path, fname)
 | 
						|
            if os.path.exists(pathname):
 | 
						|
                pathname_list.append(pathname)
 | 
						|
        return pathname_list
 | 
						|
 | 
						|
    def Scan(self, verbose):
 | 
						|
        """Scan for available toolchains and select the best for each arch.
 | 
						|
 | 
						|
        We look for all the toolchains we can file, figure out the
 | 
						|
        architecture for each, and whether it works. Then we select the
 | 
						|
        highest priority toolchain for each arch.
 | 
						|
 | 
						|
        Args:
 | 
						|
            verbose: True to print out progress information
 | 
						|
        """
 | 
						|
        if verbose: print('Scanning for tool chains')
 | 
						|
        for name, value in self.prefixes:
 | 
						|
            if verbose: print("   - scanning prefix '%s'" % value)
 | 
						|
            if os.path.exists(value):
 | 
						|
                self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
 | 
						|
                continue
 | 
						|
            fname = value + 'gcc'
 | 
						|
            if os.path.exists(fname):
 | 
						|
                self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
 | 
						|
                continue
 | 
						|
            fname_list = self.ScanPathEnv(fname)
 | 
						|
            for f in fname_list:
 | 
						|
                self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
 | 
						|
            if not fname_list:
 | 
						|
                raise ValueError("No tool chain found for prefix '%s'" %
 | 
						|
                                   value)
 | 
						|
        for path in self.paths:
 | 
						|
            if verbose: print("   - scanning path '%s'" % path)
 | 
						|
            fnames = self.ScanPath(path, verbose)
 | 
						|
            for fname in fnames:
 | 
						|
                self.Add(fname, True, verbose)
 | 
						|
 | 
						|
    def List(self):
 | 
						|
        """List out the selected toolchains for each architecture"""
 | 
						|
        col = terminal.Color()
 | 
						|
        print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
 | 
						|
                        len(self.toolchains)))
 | 
						|
        if len(self.toolchains):
 | 
						|
            for key, value in sorted(self.toolchains.items()):
 | 
						|
                print('%-10s: %s' % (key, value.gcc))
 | 
						|
        else:
 | 
						|
            print('None')
 | 
						|
 | 
						|
    def Select(self, arch):
 | 
						|
        """Returns the toolchain for a given architecture
 | 
						|
 | 
						|
        Args:
 | 
						|
            args: Name of architecture (e.g. 'arm', 'ppc_8xx')
 | 
						|
 | 
						|
        returns:
 | 
						|
            toolchain object, or None if none found
 | 
						|
        """
 | 
						|
        for tag, value in bsettings.GetItems('toolchain-alias'):
 | 
						|
            if arch == tag:
 | 
						|
                for alias in value.split():
 | 
						|
                    if alias in self.toolchains:
 | 
						|
                        return self.toolchains[alias]
 | 
						|
 | 
						|
        if not arch in self.toolchains:
 | 
						|
            raise ValueError("No tool chain found for arch '%s'" % arch)
 | 
						|
        return self.toolchains[arch]
 | 
						|
 | 
						|
    def ResolveReferences(self, var_dict, args):
 | 
						|
        """Resolve variable references in a string
 | 
						|
 | 
						|
        This converts ${blah} within the string to the value of blah.
 | 
						|
        This function works recursively.
 | 
						|
 | 
						|
        Args:
 | 
						|
            var_dict: Dictionary containing variables and their values
 | 
						|
            args: String containing make arguments
 | 
						|
        Returns:
 | 
						|
            Resolved string
 | 
						|
 | 
						|
        >>> bsettings.Setup()
 | 
						|
        >>> tcs = Toolchains()
 | 
						|
        >>> tcs.Add('fred', False)
 | 
						|
        >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
 | 
						|
                        'second' : '2nd'}
 | 
						|
        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
 | 
						|
        'this=OBLIQUE_set'
 | 
						|
        >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
 | 
						|
        'this=OBLIQUE_setfi2ndrstnd'
 | 
						|
        """
 | 
						|
        re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
 | 
						|
 | 
						|
        while True:
 | 
						|
            m = re_var.search(args)
 | 
						|
            if not m:
 | 
						|
                break
 | 
						|
            lookup = m.group(0)[2:-1]
 | 
						|
            value = var_dict.get(lookup, '')
 | 
						|
            args = args[:m.start(0)] + value + args[m.end(0):]
 | 
						|
        return args
 | 
						|
 | 
						|
    def GetMakeArguments(self, board):
 | 
						|
        """Returns 'make' arguments for a given board
 | 
						|
 | 
						|
        The flags are in a section called 'make-flags'. Flags are named
 | 
						|
        after the target they represent, for example snapper9260=TESTING=1
 | 
						|
        will pass TESTING=1 to make when building the snapper9260 board.
 | 
						|
 | 
						|
        References to other boards can be added in the string also. For
 | 
						|
        example:
 | 
						|
 | 
						|
        [make-flags]
 | 
						|
        at91-boards=ENABLE_AT91_TEST=1
 | 
						|
        snapper9260=${at91-boards} BUILD_TAG=442
 | 
						|
        snapper9g45=${at91-boards} BUILD_TAG=443
 | 
						|
 | 
						|
        This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
 | 
						|
        and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
 | 
						|
 | 
						|
        A special 'target' variable is set to the board target.
 | 
						|
 | 
						|
        Args:
 | 
						|
            board: Board object for the board to check.
 | 
						|
        Returns:
 | 
						|
            'make' flags for that board, or '' if none
 | 
						|
        """
 | 
						|
        self._make_flags['target'] = board.target
 | 
						|
        arg_str = self.ResolveReferences(self._make_flags,
 | 
						|
                           self._make_flags.get(board.target, ''))
 | 
						|
        args = re.findall("(?:\".*?\"|\S)+", arg_str)
 | 
						|
        i = 0
 | 
						|
        while i < len(args):
 | 
						|
            args[i] = args[i].replace('"', '')
 | 
						|
            if not args[i]:
 | 
						|
                del args[i]
 | 
						|
            else:
 | 
						|
                i += 1
 | 
						|
        return args
 | 
						|
 | 
						|
    def LocateArchUrl(self, fetch_arch):
 | 
						|
        """Find a toolchain available online
 | 
						|
 | 
						|
        Look in standard places for available toolchains. At present the
 | 
						|
        only standard place is at kernel.org.
 | 
						|
 | 
						|
        Args:
 | 
						|
            arch: Architecture to look for, or 'list' for all
 | 
						|
        Returns:
 | 
						|
            If fetch_arch is 'list', a tuple:
 | 
						|
                Machine architecture (e.g. x86_64)
 | 
						|
                List of toolchains
 | 
						|
            else
 | 
						|
                URL containing this toolchain, if avaialble, else None
 | 
						|
        """
 | 
						|
        arch = command.OutputOneLine('uname', '-m')
 | 
						|
        if arch == 'aarch64':
 | 
						|
            arch = 'arm64'
 | 
						|
        base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
 | 
						|
        versions = ['11.1.0', '9.2.0', '7.3.0', '6.4.0', '4.9.4']
 | 
						|
        links = []
 | 
						|
        for version in versions:
 | 
						|
            url = '%s/%s/%s/' % (base, arch, version)
 | 
						|
            print('Checking: %s' % url)
 | 
						|
            response = urllib.request.urlopen(url)
 | 
						|
            html = tools.ToString(response.read())
 | 
						|
            parser = MyHTMLParser(fetch_arch)
 | 
						|
            parser.feed(html)
 | 
						|
            if fetch_arch == 'list':
 | 
						|
                links += parser.links
 | 
						|
            elif parser.arch_link:
 | 
						|
                return url + parser.arch_link
 | 
						|
        if fetch_arch == 'list':
 | 
						|
            return arch, links
 | 
						|
        return None
 | 
						|
 | 
						|
    def Download(self, url):
 | 
						|
        """Download a file to a temporary directory
 | 
						|
 | 
						|
        Args:
 | 
						|
            url: URL to download
 | 
						|
        Returns:
 | 
						|
            Tuple:
 | 
						|
                Temporary directory name
 | 
						|
                Full path to the downloaded archive file in that directory,
 | 
						|
                    or None if there was an error while downloading
 | 
						|
        """
 | 
						|
        print('Downloading: %s' % url)
 | 
						|
        leaf = url.split('/')[-1]
 | 
						|
        tmpdir = tempfile.mkdtemp('.buildman')
 | 
						|
        response = urllib.request.urlopen(url)
 | 
						|
        fname = os.path.join(tmpdir, leaf)
 | 
						|
        fd = open(fname, 'wb')
 | 
						|
        meta = response.info()
 | 
						|
        size = int(meta.get('Content-Length'))
 | 
						|
        done = 0
 | 
						|
        block_size = 1 << 16
 | 
						|
        status = ''
 | 
						|
 | 
						|
        # Read the file in chunks and show progress as we go
 | 
						|
        while True:
 | 
						|
            buffer = response.read(block_size)
 | 
						|
            if not buffer:
 | 
						|
                print(chr(8) * (len(status) + 1), '\r', end=' ')
 | 
						|
                break
 | 
						|
 | 
						|
            done += len(buffer)
 | 
						|
            fd.write(buffer)
 | 
						|
            status = r'%10d MiB  [%3d%%]' % (done // 1024 // 1024,
 | 
						|
                                             done * 100 // size)
 | 
						|
            status = status + chr(8) * (len(status) + 1)
 | 
						|
            print(status, end=' ')
 | 
						|
            sys.stdout.flush()
 | 
						|
        fd.close()
 | 
						|
        if done != size:
 | 
						|
            print('Error, failed to download')
 | 
						|
            os.remove(fname)
 | 
						|
            fname = None
 | 
						|
        return tmpdir, fname
 | 
						|
 | 
						|
    def Unpack(self, fname, dest):
 | 
						|
        """Unpack a tar file
 | 
						|
 | 
						|
        Args:
 | 
						|
            fname: Filename to unpack
 | 
						|
            dest: Destination directory
 | 
						|
        Returns:
 | 
						|
            Directory name of the first entry in the archive, without the
 | 
						|
            trailing /
 | 
						|
        """
 | 
						|
        stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
 | 
						|
        dirs = stdout.splitlines()[1].split('/')[:2]
 | 
						|
        return '/'.join(dirs)
 | 
						|
 | 
						|
    def TestSettingsHasPath(self, path):
 | 
						|
        """Check if buildman will find this toolchain
 | 
						|
 | 
						|
        Returns:
 | 
						|
            True if the path is in settings, False if not
 | 
						|
        """
 | 
						|
        paths = self.GetPathList(False)
 | 
						|
        return path in paths
 | 
						|
 | 
						|
    def ListArchs(self):
 | 
						|
        """List architectures with available toolchains to download"""
 | 
						|
        host_arch, archives = self.LocateArchUrl('list')
 | 
						|
        re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
 | 
						|
        arch_set = set()
 | 
						|
        for archive in archives:
 | 
						|
            # Remove the host architecture from the start
 | 
						|
            arch = re_arch.match(archive[len(host_arch):])
 | 
						|
            if arch:
 | 
						|
                if arch.group(1) != '2.0' and arch.group(1) != '64':
 | 
						|
                    arch_set.add(arch.group(1))
 | 
						|
        return sorted(arch_set)
 | 
						|
 | 
						|
    def FetchAndInstall(self, arch):
 | 
						|
        """Fetch and install a new toolchain
 | 
						|
 | 
						|
        arch:
 | 
						|
            Architecture to fetch, or 'list' to list
 | 
						|
        """
 | 
						|
        # Fist get the URL for this architecture
 | 
						|
        col = terminal.Color()
 | 
						|
        print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
 | 
						|
        url = self.LocateArchUrl(arch)
 | 
						|
        if not url:
 | 
						|
            print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
 | 
						|
                   arch))
 | 
						|
            return 2
 | 
						|
        home = os.environ['HOME']
 | 
						|
        dest = os.path.join(home, '.buildman-toolchains')
 | 
						|
        if not os.path.exists(dest):
 | 
						|
            os.mkdir(dest)
 | 
						|
 | 
						|
        # Download the tar file for this toolchain and unpack it
 | 
						|
        tmpdir, tarfile = self.Download(url)
 | 
						|
        if not tarfile:
 | 
						|
            return 1
 | 
						|
        print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
 | 
						|
        sys.stdout.flush()
 | 
						|
        path = self.Unpack(tarfile, dest)
 | 
						|
        os.remove(tarfile)
 | 
						|
        os.rmdir(tmpdir)
 | 
						|
        print()
 | 
						|
 | 
						|
        # Check that the toolchain works
 | 
						|
        print(col.Color(col.GREEN, 'Testing'))
 | 
						|
        dirpath = os.path.join(dest, path)
 | 
						|
        compiler_fname_list = self.ScanPath(dirpath, True)
 | 
						|
        if not compiler_fname_list:
 | 
						|
            print('Could not locate C compiler - fetch failed.')
 | 
						|
            return 1
 | 
						|
        if len(compiler_fname_list) != 1:
 | 
						|
            print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
 | 
						|
                            ', '.join(compiler_fname_list)))
 | 
						|
        toolchain = Toolchain(compiler_fname_list[0], True, True)
 | 
						|
 | 
						|
        # Make sure that it will be found by buildman
 | 
						|
        if not self.TestSettingsHasPath(dirpath):
 | 
						|
            print(("Adding 'download' to config file '%s'" %
 | 
						|
                   bsettings.config_fname))
 | 
						|
            bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
 | 
						|
        return 0
 |