buildman improvements including toolchain environment feature

sandbox unicode support in serial
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCgAdFiEEslwAIq+Gp8wWVbYnfxc6PpAIreYFAl3xP/0ACgkQfxc6PpAI
 reYTTAf/Uuhm4meLxXyho3zelrYZKcVtLcbQSTwwMKsfQYV/rH+Hbi7StbQSO1+K
 UYt/A2O0ic0U+UiSpHv8A5n2JGzcmeJK1f6BJvyoXAhXJAxRKGQhxAAE+X/GcSi8
 8gpfYEAb0Zpd+1eTBBMSWLXrFAwITbfqe45L4BcDb9BT+Dkhidr9SZQahu8xzgMa
 Wf0oXUQy+HmHB6YYTUcpyUisYJV5SiuVZDqsMDJ58T/4nYKnAV2qlu647YFiWyZe
 A+EeL13LbAGHibz385J5rAYb6RFM8SlWunicQnFrsYHoagfxTrrnkwF5EZdeBb1e
 sCX+jNk28WbjcAN6wjDcOSKIVDQ4wQ==
 =Ycht
 -----END PGP SIGNATURE-----

Merge tag 'dm-next-13dec19' of https://gitlab.denx.de/u-boot/custodians/u-boot-dm into next

buildman improvements including toolchain environment feature
sandbox unicode support in serial

arch/sandbox/dts/sandbox.dts

@@ -6,6 +6,7 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 	model = "sandbox";
+	compatible = "sandbox";
 
 	aliases {
 		i2c0 = &i2c_0;

arch/sandbox/dts/sandbox64.dts

@@ -6,6 +6,7 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 	model = "sandbox";
+	compatible = "sandbox";
 
 	aliases {
 		i2c0 = &i2c_0;

drivers/misc/cros_ec.c

@@ -313,7 +313,7 @@ static int send_command(struct cros_ec_dev *dev, uint cmd, int cmd_version,
  * @param din_len       Maximum size of response in bytes
  * @return number of bytes in response, or -ve on error
  */
-static int ec_command_inptr(struct udevice *dev, uint8_t cmd,
+static int ec_command_inptr(struct udevice *dev, uint cmd,
 			    int cmd_version, const void *dout, int dout_len,
 			    uint8_t **dinp, int din_len)
 {

drivers/serial/sandbox.c

@@ -33,7 +33,7 @@ DECLARE_GLOBAL_DATA_PTR;
  *   serial_buf_write		 == serial_buf_read -> empty buffer
  *   (serial_buf_write + 1) % 16 == serial_buf_read -> full buffer
  */
-static char serial_buf[16];
+static unsigned char serial_buf[16];
 static unsigned int serial_buf_write;
 static unsigned int serial_buf_read;
 

test/py/conftest.py

@@ -83,6 +83,26 @@ def pytest_configure(config):
     Returns:
         Nothing.
     """
+    def parse_config(conf_file):
+        """Parse a config file, loading it into the ubconfig container
+
+        Args:
+            conf_file: Filename to load (within build_dir)
+
+        Raises
+            Exception if the file does not exist
+        """
+        dot_config = build_dir + '/' + conf_file
+        if not os.path.exists(dot_config):
+            raise Exception(conf_file + ' does not exist; ' +
+                            'try passing --build option?')
+
+        with open(dot_config, 'rt') as f:
+            ini_str = '[root]\n' + f.read()
+            ini_sio = io.StringIO(ini_str)
+            parser = configparser.RawConfigParser()
+            parser.read_file(ini_sio)
+            ubconfig.buildconfig.update(parser.items('root'))
 
     global log
     global console
@@ -157,18 +177,13 @@ def pytest_configure(config):
 
     ubconfig.buildconfig = dict()
 
-    for conf_file in ('.config', 'include/autoconf.mk'):
+    # buildman -k puts autoconf.mk in the rootdir, so handle this as well
-        dot_config = build_dir + '/' + conf_file
+    # as the standard U-Boot build which leaves it in include/autoconf.mk
-        if not os.path.exists(dot_config):
+    parse_config('.config')
-            raise Exception(conf_file + ' does not exist; ' +
+    if os.path.exists(build_dir + '/' + 'autoconf.mk'):
-                'try passing --build option?')
+        parse_config('autoconf.mk')
-
+    else:
-        with open(dot_config, 'rt') as f:
+        parse_config('include/autoconf.mk')
-            ini_str = '[root]\n' + f.read()
-            ini_sio = io.StringIO(ini_str)
-            parser = configparser.RawConfigParser()
-            parser.read_file(ini_sio)
-            ubconfig.buildconfig.update(parser.items('root'))
 
     ubconfig.test_py_dir = test_py_dir
     ubconfig.source_dir = source_dir

tools/buildman/README

@@ -1061,6 +1061,9 @@ Other options
 
 Buildman has various other command line options. Try --help to see them.
 
+To find out what architecture or toolchain prefix buildman will use for a build,
+see the -a and -A options.
+
 When doing builds, Buildman's return code will reflect the overall result:
 
     0 (success)     No errors or warnings found

tools/buildman/builder.py

@@ -577,6 +577,7 @@ class Builder:
         sym = {}
         for line in fd.readlines():
             try:
+                if line.strip():
                     size, type, name = line[:-1].split()
             except:
                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))

tools/buildman/cmdline.py

@@ -13,6 +13,10 @@ def ParseArgs():
             args: command lin arguments
     """
     parser = OptionParser()
+    parser.add_option('-a', '--print-arch', action='store_true',
+          help='Print the architecture for a board (ARCH=)')
+    parser.add_option('-A', '--print-prefix', action='store_true',
+          help='Print the tool-chain prefix for a board (CROSS_COMPILE=)')
     parser.add_option('-b', '--branch', type='string',
           help='Branch name to build, or range of commits to build')
     parser.add_option('-B', '--bloat', dest='show_bloat',

tools/buildman/control.py

@@ -107,6 +107,34 @@ def CheckOutputDir(output_dir):
             break
         path = parent
 
+def ShowToolchainInfo(boards, toolchains, print_arch, print_prefix):
+    """Show information about a the tool chain used by one or more boards
+
+    The function checks that all boards use the same toolchain.
+
+    Args:
+        boards: Boards object containing selected boards
+        toolchains: Toolchains object containing available toolchains
+        print_arch: True to print ARCH value
+        print_prefix: True to print CROSS_COMPILE value
+
+    Return:
+        None on success, string error message otherwise
+    """
+    boards = boards.GetSelectedDict()
+    tc_set = set()
+    for brd in boards.values():
+        tc_set.add(toolchains.Select(brd.arch))
+    if len(tc_set) != 1:
+        return 'Supplied boards must share one toolchain'
+        return False
+    tc = tc_set.pop()
+    if print_arch:
+        print(tc.GetEnvArgs(toolchain.VAR_ARCH))
+    if print_prefix:
+        print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
+    return None
+
 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
                clean_dir=False):
     """The main control code for buildman
@@ -170,6 +198,43 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
         print()
         return 0
 
+    # Work out what subset of the boards we are building
+    if not boards:
+        if not os.path.exists(options.output_dir):
+            os.makedirs(options.output_dir)
+        board_file = os.path.join(options.output_dir, 'boards.cfg')
+        genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
+        status = subprocess.call([genboardscfg, '-q', '-o', board_file])
+        if status != 0:
+            sys.exit("Failed to generate boards.cfg")
+
+        boards = board.Boards()
+        boards.ReadBoards(board_file)
+
+    exclude = []
+    if options.exclude:
+        for arg in options.exclude:
+            exclude += arg.split(',')
+
+    if options.boards:
+        requested_boards = []
+        for b in options.boards:
+            requested_boards += b.split(',')
+    else:
+        requested_boards = None
+    why_selected, board_warnings = boards.SelectBoards(args, exclude,
+                                                       requested_boards)
+    selected = boards.GetSelected()
+    if not len(selected):
+        sys.exit(col.Color(col.RED, 'No matching boards found'))
+
+    if options.print_arch or options.print_prefix:
+        err = ShowToolchainInfo(boards, toolchains, options.print_arch,
+                                options.print_prefix)
+        if err:
+            sys.exit(col.Color(col.RED, err))
+        return 0
+
     # Work out how many commits to build. We want to build everything on the
     # branch. We also build the upstream commit as a control so we can see
     # problems introduced by the first commit on the branch.
@@ -199,37 +264,6 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
                "set branch's upstream or use -c flag" % options.branch)
         sys.exit(col.Color(col.RED, str))
 
-    # Work out what subset of the boards we are building
-    if not boards:
-        if not os.path.exists(options.output_dir):
-            os.makedirs(options.output_dir)
-        board_file = os.path.join(options.output_dir, 'boards.cfg')
-        genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
-        status = subprocess.call([genboardscfg, '-o', board_file])
-        if status != 0:
-            sys.exit("Failed to generate boards.cfg")
-
-        boards = board.Boards()
-        boards.ReadBoards(board_file)
-
-    exclude = []
-    if options.exclude:
-        for arg in options.exclude:
-            exclude += arg.split(',')
-
-
-    if options.boards:
-        requested_boards = []
-        for b in options.boards:
-            requested_boards += b.split(',')
-    else:
-        requested_boards = None
-    why_selected, board_warnings = boards.SelectBoards(args, exclude,
-                                                       requested_boards)
-    selected = boards.GetSelected()
-    if not len(selected):
-        sys.exit(col.Color(col.RED, 'No matching boards found'))
-
     # Read the metadata from the commits. First look at the upstream commit,
     # then the ones in the branch. We would like to do something like
     # upstream/master~..branch but that isn't possible if upstream/master is

tools/buildman/test.py

@@ -451,6 +451,24 @@ class TestBuild(unittest.TestCase):
                     'crosstool/files/bin/x86_64/.*/'
                     'x86_64-gcc-.*-nolibc_arm-.*linux-gnueabi.tar.xz')
 
+    def testGetEnvArgs(self):
+        """Test the GetEnvArgs() function"""
+        tc = self.toolchains.Select('arm')
+        self.assertEqual('arm-linux-',
+                         tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
+        self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_PATH))
+        self.assertEqual('arm',
+                         tc.GetEnvArgs(toolchain.VAR_ARCH))
+        self.assertEqual('', tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS))
+
+        self.toolchains.Add('/path/to/x86_64-linux-gcc', test=False)
+        tc = self.toolchains.Select('x86')
+        self.assertEqual('/path/to',
+                         tc.GetEnvArgs(toolchain.VAR_PATH))
+        tc.override_toolchain = 'clang'
+        self.assertEqual('HOSTCC=clang CC=clang',
+                         tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS))
+
 
 if __name__ == "__main__":
     unittest.main()

tools/buildman/toolchain.py

@@ -18,6 +18,8 @@ import tools
 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
     PRIORITY_CALC) = list(range(4))
 
+(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
+
 # Simple class to collect links from a page
 class MyHTMLParser(HTMLParser):
     def __init__(self, arch):
@@ -145,6 +147,30 @@ class Toolchain:
 
         return value
 
+    def GetEnvArgs(self, which):
+        """Get an environment variable/args value based on the the toolchain
+
+        Args:
+            which: VAR_... value to get
+
+        Returns:
+            Value of that environment variable or arguments
+        """
+        wrapper = self.GetWrapper()
+        if which == VAR_CROSS_COMPILE:
+            return wrapper + os.path.join(self.path, self.cross)
+        elif which == VAR_PATH:
+            return self.path
+        elif which == VAR_ARCH:
+            return self.arch
+        elif which == VAR_MAKE_ARGS:
+            args = self.MakeArgs()
+            if args:
+                return ' '.join(args)
+            return ''
+        else:
+            raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
+
     def MakeEnvironment(self, full_path):
         """Returns an environment for using the toolchain.
 
@@ -435,9 +461,10 @@ class Toolchains:
         self._make_flags['target'] = board.target
         arg_str = self.ResolveReferences(self._make_flags,
                            self._make_flags.get(board.target, ''))
-        args = arg_str.split(' ')
+        args = re.findall("(?:\".*?\"|\S)+", arg_str)
         i = 0
         while i < len(args):
+            args[i] = args[i].replace('"', '')
             if not args[i]:
                 del args[i]
             else:

tools/genboardscfg.py

@@ -403,17 +403,19 @@ def format_and_output(params_list, output):
     with open(output, 'w', encoding="utf-8") as f:
         f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
 
-def gen_boards_cfg(output, jobs=1, force=False):
+def gen_boards_cfg(output, jobs=1, force=False, quiet=False):
     """Generate a board database file.
 
     Arguments:
       output: The name of the output file
       jobs: The number of jobs to run simultaneously
       force: Force to generate the output even if it is new
+      quiet: True to avoid printing a message if nothing needs doing
     """
     check_top_directory()
 
     if not force and output_is_new(output):
+        if not quiet:
             print("%s is up to date. Nothing to do." % output)
         sys.exit(0)
 
@@ -435,9 +437,11 @@ def main():
                       help='the number of jobs to run simultaneously')
     parser.add_option('-o', '--output', default=OUTPUT_FILE,
                       help='output file [default=%s]' % OUTPUT_FILE)
+    parser.add_option('-q', '--quiet', action="store_true", help='run silently')
     (options, args) = parser.parse_args()
 
-    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
+    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force,
+                   quiet=options.quiet)
 
 if __name__ == '__main__':
     main()