Pull request #9: feat: allow CoreOS to be embedded in another repository

Merge in ICO/coreos from feat/embeding-coreos to master

* commit 'e221f49cbf50aae44471b4f82766aa3f4121905b':
  feat: allow CoreOS to be embedded in another repository

.vscode/extensions.json

@@ -0,0 +1,6 @@
+{
+    "recommendations": [
+        "ms-vscode.makefile-tools",
+        "timonwong.shellcheck"
+    ]
+}

coreos-init-build-env

@@ -3,53 +3,83 @@
 # This script is used to setup the OE Build Envrionment
 # Normally this is called as '. ./core-init-build-env <builddir>'
 
+# Configuration variables
+# ------------------------------------------------------------------------------
+# We only set default value, so all configuration variable can be overriden
+# if already set when sourcing this script
+
 # On some shell, we can get the path of this script when sources. Otherwise we
 # use the current directory as a fallback
-if [ -n "$BASH_SOURCE" ]; then
+if [ -z "$COREOS_ROOT" ]; then
-    CORE_OS_ROOT=$(dirname "$BASH_SOURCE")
+    if [ -n "$BASH_SOURCE" ]; then
-elif [ -n "$ZSH_NAME" ]; then
+        COREOS_ROOT=$(dirname "$BASH_SOURCE")
-    CORE_OS_ROOT=$(dirname "$0")
+    elif [ -n "$ZSH_NAME" ]; then
-else
+        COREOS_ROOT=$(dirname "$0")
-    CORE_OS_ROOT="$(pwd)"
+    else
+        COREOS_ROOT="$(pwd)"
+    fi
 fi
 
 # Get a non relative path to the root directory
-CORE_OS_ROOT=$(readlink -f "${CORE_OS_ROOT}")
+COREOS_ROOT=$(readlink -f "${COREOS_ROOT}")
 
 # Set the path to bitbake, openembedded-core and the template directory
-BITBAKEDIR="${CORE_OS_ROOT}/bitbake"
+# All theses values can be overriden by the caller of coreos-init-build-env
-OEROOT="${CORE_OS_ROOT}/layers/openembedded-core"
+BITBAKEDIR="${BITBAKEDIR:-${COREOS_ROOT}/bitbake}"
-TEMPLATECONF="${CORE_OS_ROOT}/templates"
+OEROOT="${OEROOT:-${COREOS_ROOT}/layers/openembedded-core}"
+TEMPLATECONF="${TEMPLATECONF:-${COREOS_ROOT}/templates}"
 
 # Sanity checks
+# ------------------------------------------------------------------------------
+
+# BITBAKEDIR, OEROOT, TEMPLATECONF and COREOS_ROOT can be overridden by our user
+# so let's check that they have valid value
+
+if [ ! -f "${COREOS_ROOT}/coreos-init-build-env" ]; then
+    echo "Error: COREOS_ROOT ($COREOS_ROOT) isn't valid" >&2
+    echo "If you are using CoreOS directly, try using this script from CoreOS root directory." >&2
+    echo "If you are embedding coreos-init-build-env in another script, set COREOS_ROOT correctly there." >&2
+    return 1
+fi
 
-# This check detect if CORE_OS_ROOT is valid, by checking if the templates
-# directory exists. Usefull has we use $(pwd) as a fallback method on some shell
 if [ ! -d "$TEMPLATECONF" ]; then
-    echo "Error: $TEMPLATECONF doesn't exist!" >&2
+    echo "Error: TEMPLATECONF (${TEMPLATECONF}) doesn't exist!" >&2
-    echo "Please run this script in oe-init-build-env's directory." >&2
+    echo "Please check your TEMPLATECONF configuration." >&2
     return 1
 fi
 
-# This check detect if BITBAKEDIR exist. It's a simple way to check that we have
+if [ ! -f "${BITBAKEDIR}/bin/bitbake" ]; then
-# fetched our git submodules
+    echo "Error: BITBAKEDIR (${BITBAKEDIR}) isn't valid!" >&2
-if [ ! -d "$BITBAKEDIR" ]; then
-    echo "Error: $BITBAKEDIR doesn't exist!" >&2
     echo "Please ensure all git submodule are fetched." >&2
+    echo "And check your BITBAKEDIR configuration." >&2
     return 1
 fi
 
+if [ ! -f "${OEROOT}/oe-init-build-env" ]; then
+    echo "Error: OEROOT (${OEROOT}) isn't valid!" >&2
+    echo "Please ensure all git submodule are fetched." >&2
+    echo "And check your OEROOT configuration." >&2
+    return 1
+fi
+
+# Build environmnet setup
+# ------------------------------------------------------------------------------
+
 # Call the oe-init-build-env scripts of openembedded-core
-. "${OEROOT}/oe-init-build-env" "${1:-$CORE_OS_ROOT/build}"
+. "${OEROOT}/oe-init-build-env" "${1:-$COREOS_ROOT/build}"
 
-# Add the scripts directory of CoreOS to the path
+# Add the first argument of the function to the path
-# Make sure our paths are at the beginning of $PATH
+coreos_path_add() {
-for newpath in "${CORE_OS_ROOT}/scripts"; do
+    # Make sure our paths are at the beginning of $PATH
-    # Remove any existences of $newpath from $PATH
+    # Remove any existences of $1 from $PATH
-    PATH=$(echo $PATH | sed -re "s#(^|:)$newpath(:|$)#\2#g;s#^:##")
+    PATH=$(echo "$PATH" | sed -re "s#(^|:)$1(:|$)#\2#g;s#^:##")
 
-    # Add $newpath to $PATH
+    # Add $1 to the PATH
-    PATH="$newpath:$PATH"
+    PATH="$1:$PATH"
-done
+    export PATH
-unset newpath
+}
-export PATH
+
+coreos_path_add "${COREOS_ROOT}/scripts"
+
+# Add support for ##COREOS_LAYERSDIR## inside of bblayer template
+coreos-bblayers-envsub COREOS_LAYERSDIR "${COREOS_ROOT}/layers"

documentation/conf.py

@@ -40,6 +40,7 @@ version = release
 extensions = [
     'sphinx.ext.extlinks',
     'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
 ]
 
 # external links and substitutions
@@ -77,7 +78,7 @@ templates_path = ['_templates']
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'oe-workdir', 'oe-logs']
 
 
 # -- Options for HTML output -------------------------------------------------

documentation/index.rst

@@ -22,6 +22,7 @@ same structures.
    :caption: Introduction and Overview
 
    Quick Build <quick-build>
+   Setting up a CoreOS based distro <using-coreos>
 
 .. toctree::
    :maxdepth: 1

documentation/ref-manual/classes.rst

@@ -3,7 +3,8 @@ Classes
 *******
 
 This chapter document the classes that are provided by Belden CoreOS. Classes
-provided by OpenEmbedded-Core are documented in the Yocto Reference Manual.
+provided by OpenEmbedded-Core are documented in the
+:external:doc:`Yocto Reference Manual <ref-manual/classes>`.
 
 .. _ref-classes-coreos-metadata-scm:
 .. index:: coreos_metadata_scm.bbclass
@@ -12,10 +13,9 @@ provided by OpenEmbedded-Core are documented in the Yocto Reference Manual.
 ===============================
 
 The ``coreos_metadata_scm`` class is used inside the CoreOS distribution
-configuration file to change the value of ``METADATA_BRANCH`` and
+configuration file to set the variables ``COREOS_METADATA_BRANCH`` and
-``METADATA_REVISION`` to the current Git branch and revision of the main CoreOS
+``COREOS_METADATA_REVISION`` to the current Git branch and revision of the main
-repository instead of the branch and revision of the OpenEmbedded-Core Layer, as
+CoreOS repository.
-set by the :external:ref:`metadata_scm <ref-classes-metadata_scm>` class.
 
 The ``coreos_metadata_scm`` is automatically inherited if ``DISTRO`` is set to
 ``belden-coreos`` or to any distro based on ``belden-coreos``.
@@ -34,7 +34,7 @@ The ``coreos-image`` class provides common definitions for the
 .. index:: coreos-sanity.class
 
 ``coreos-sanity.bbclass``
-========================
+=========================
 
 The ``coreos-sanity`` class is inherited inside the CoreOS layer
 configuration file to add some sanity checks. Theses check ensure that the 

documentation/ref-manual/index.rst

@@ -13,3 +13,4 @@ Belden CoreOS Reference Manual
    distro
    images
    features
+   variables

documentation/ref-manual/variables.rst

@@ -0,0 +1,31 @@
+
+******************
+Variables Glossary
+******************
+
+This chapter lists common variables used in the CoreOS build
+system and gives an overview of their function and contents.
+
+Variables provided by OpenEmbedded-Core are documented in the
+:external:doc:`Yocto Reference Manual <ref-manual/variables>`.
+
+
+.. glossary::
+    :sorted:
+
+    :term:`COREOS_ROOT`
+
+        Specifies the root directory of CoreOS.
+
+        It is an important distinction that :term:`COREOS_ROOT` points to root of
+        the Git repository of CoreOS, and not to a layer.
+
+    :term:`COREOS_METADATA_BRANCH`
+
+        The branch currently checked out for the CoreOS project (path
+        determined by :term:`COREOS_ROOT`).
+    
+    :term:`COREOS_METADATA_REVISION`
+
+        The revision currently checked out for the CoreOS project (path
+        determined by :term:`COREOS_ROOT`).

documentation/using-coreos.rst

@@ -0,0 +1,249 @@
+********************************
+Setting up a CoreOS based distro
+********************************
+
+This chapter explain how to setup a distro based in CoreOS
+
+Repository structures
+#####################
+
+OpenEmbedded is a flexible tools, but we encourage each of our users to adopt
+the same structure as CoreOS. In this chapter, replace each usage of `PRODUCT`
+or `product` by a unique name related to your product.
+
+.. code-block::
+
+    product/
+    ├── build/ (ignored by git)
+    ├── documentation/
+    ├── layers/
+    |   └── coreos/ (submodule)
+    |   |   └── bitbake/ (submodule)
+    |   |   └── layers/
+    |   |   |   ├── openembedded-core (submodule)
+    |   |   |   ├── meta-belden-coreos
+    |   |   |   ├── meta-belden-coreos-bsp
+    |   |   |   └── ...
+    |   |   └── ...
+    |   ├── meta-product/
+    |   ├── meta-other-layers/
+    |   └── ...
+    ├── scripts/
+    ├── templates/
+    ├── product-init-build-env
+    ├── .gitignore
+
+Creating the structures
+#######################
+
+.. code-block:: sh
+
+    ~$  mkdir product
+    ~$  cd product
+    ~/product$ git init
+    ~/product$ git submodule init
+
+    ~/product$ mkdir layers
+    ~/product$ mkdir scripts
+
+    ~/product$ cd layers
+    ~/product/layers$ git clone --recurse-submodules ssh://git@bitbucket.gad.local:7999/ico/coreos.git
+
+    ~/product/layers$ cd ..
+    ~/product$ cp -r layers/coreos/templates ./templates
+    ~/product$ cp layers/coreos/.gitignore ./.gitignore
+
+    ~/product$ touch product-init-build-env
+    ~/product$ chmod +x product-init-build-env
+    ~/product$ nano product-init-build-env
+
+.. note::
+
+    By copying the .gitignore file of CoreOS, the build directory of the product
+    will not be stored inside of Git, which is the recommended approach as using
+    `devtool modify` modify the local `bblayers.conf`. Instead we recommend to
+    keep the template directory up to date so that a sane configuration can be
+    created when fetching the repository for the first time.
+
+Then you can enter the following inside the product-init-build-env files
+
+.. code-block:: sh
+
+    #!/bin/sh
+
+    # This script is used to setup the OE Build Envrionment
+    # Normally this is called as '. ./product-init-build-env <builddir>'
+
+    # On some shell, we can get the path of this script when sources. Otherwise we
+    # use the current directory as a fallback
+    if [ -z "$PRODUCT_ROOT" ]; then
+        if [ -n "$BASH_SOURCE" ]; then
+            PRODUCT_ROOT=$(dirname "$BASH_SOURCE")
+        elif [ -n "$ZSH_NAME" ]; then
+            PRODUCT_ROOT=$(dirname "$0")
+        else
+            PRODUCT_ROOT="$(pwd)"
+        fi
+    fi
+
+    # Get a non relative path to the root directory
+    PRODUCT_ROOT=$(readlink -f "${PRODUCT_ROOT}")
+
+    # CoreOS init settings
+    COREOS_ROOT="${PRODUCT_ROOT}/layers/coreos"
+    TEMPLATECONF="${PRODUCT_ROOT}/templates"
+
+    # Call the coreos-init-build-env scripts of CoreOS
+    . "${COREOS_ROOT}/coreos-init-build-env" "${1:-$PRODUCT_ROOT/build}"
+
+    # From here the scripts and functions defined by CoreOS and
+    # OpenEmbedded-Core are available
+
+    # Add support for ##PRODUCTS_LAYERSDIR## inside of bblayer template
+    coreos-bblayers-envsub PRODUCT_LAYERSDIR "${PRODUCT_ROOT}/layers"
+
+    # Add the scripts directory of the product to the path
+    coreos_path_add "${PRODUCT_ROOT}/scripts"
+
+Using you new projects
+######################
+
+.. code-block:: sh
+
+    ~product$  source product-init-build-env
+
+Creating your product layers
+############################
+
+You can create a new layers and add it to your active bblayers.conf file like
+this:
+
+.. code-block:: sh
+
+    ~product/build$ bitbake-layers create-layer ../layers/meta-belden-bsp
+    ~product/build$ bitbake-layers add-layer ../layers/meta-product
+
+Don't forget to update your templates `projects/templates/bblayers.conf.sample`
+file. Inside this file use `##PRODUCT_LAYERSDIR##/meta-product` to have a
+machine agnostic path.
+
+Optional: Change some git settings
+##################################
+
+If you want to always `--recurse-submodules` when using `git pull`, you can
+change your `submodule.recurse` git setting, either locally or globally
+
+.. code-block:: sh
+
+    ~/product$ git config submodule.recurse true # Only inside of product repo
+    ~/product$ git config --global submodule.recurse true # Set it for all repos
+
+
+Create your own distro based on CoreOS
+######################################
+
+Create a new file inside configuration file inside
+`product/layers/meta-product/conf/distro`. For a distro named, you will create
+`product/layers/meta-product/conf/distro/product.conf`.
+
+Open this file and enter the following:
+
+.. code-block:: ini
+
+    # This should come at the beginning of the file, to ensure that you use
+    # CoreOS defaults
+    require conf/distro/belden-coreos.conf
+
+    # This should always be set in your own configuration file, to not use the
+    # value of CoreOS
+    DISTRO = "product"
+    DISTRO_NAME = "Product Linux Distribution"
+    MAINTAINER = "Belden Product Team"
+
+    # You may want to add a version and a codename to your distro instead of
+    # using the version and codename of CoreOS
+    DISTRO_VERSION = "2022.05"
+    DISTRO_CODENAME = "ProductOS Summer 2022 Edition"
+
+    # From here you can overrides settings from the CoreOS distro or from
+    # OpenEmbedded-core. But keep in mind that the CoreOS teams doesn't support
+    # all the features of OpenEmbedded-Core. We have added some checks for some
+    # of the settings that we don't allow to change or that we don't support.
+    # See the coreos-sanity.bbclass file for more info.
+
+Then you can activate the distro by setting the `DISTRO` to `product` inside
+your `product/build/conf/local.conf` file. You should also set it in the 
+`product/templates/local.conf.sample` file so that it will be set as the default
+when create the build envrionment for the first time.
+
+What to do next
+###############
+
+How do I...
+-----------
+
+**...add a PRODUCT_ROOT variable usable in recipes files?**
+
+Add this line inside your meta-product layer configuration file at 
+`product/layers/meta-product/conf/layer.conf`:
+
+.. code-block:: ini
+
+    # Set a variable to get to the top of the metadata location
+    PRODUCT_ROOT = '${@os.path.normpath("${LAYERDIR}/../../")}'
+
+**... add PRODUCT_METADATA_BRANCH and PRODUCT_METADATA_REVISION variables to get the
+current git branch and git sha of the PRODUCT repository?**
+    
+Create the file `product/layers/meta-product/classes/product_metadata_scm.bbclass`
+and copy the content of the coreos_metadata_scm.bbclass file. Replacing all
+reference to COREOS by PRODUCT should works.
+
+Alternative repository structure
+################################
+
+It's also possible but not recommended to clone CoreOS without any submodule, to
+create a more flat structure. But then you have to ensure and manage the 
+Bitbake et OpenEmbedded-Core version by yourself.
+
+.. important::
+
+    CoreOS is only tested with the version of Bitbake and OpenEmbedded-Core used
+    in the CoreOS repository as submodule. By doing this you have to ensure that
+    you project stay in sync with CoreOS regarding CoreOS version and
+    corresponding Bitbake and OpenEmbedded-Core version.
+
+
+.. code-block::
+
+    product/
+    ├── build/ (ignored by git)
+    ├── bitbake/ (submodule)
+    ├── documentation/
+    ├── layers/
+    |   ├── openembedded-core (submodule)
+    |   └── coreos/ (cloned without submodule)
+    |   |   ├── layers/
+    |   |   |   ├── meta-belden-coreos
+    |   |   |   ├── meta-belden-coreos-bsp
+    |   |   |   └── ...
+    |   |   └── ...
+    |   ├── meta-product/
+    |   ├── meta-other-layers/
+    |   └── ...
+    ├── scripts/
+    ├── templates/
+    ├── product-init-build-env
+    ├── .gitignore
+
+Setting this structure is out of the scope for this documentation, but as a
+hint, to implement it you have to set in `product-init-build-env`:
+
+- `BITBAKEDIR` to the path of the Bitbake repository
+- `OEROOT` to the path of the OpenEmbedded-Core repository
+
+.. important::
+
+    Calling directly oe-init-build-env from OpenEmbedded-Core is not supported!
+    Ensure that your product-init-build-env call coreos-init-build-env egal if
+    you use the recommended or alternative repository structures.

layers/meta-belden-coreos/classes/coreos_metadata_scm.bbclass

@@ -1,9 +1,11 @@
-# This change the default value of variables defined into metadata_scm.bbclass
+# This create the same variable for CoreOS as defined by metadata_scm.bbclass
+# for OpenEmbedded-Core
+
 # Note that metadata_scm.bbclass is already ihnerited by the base class, so we
 # don't need to add it here.
 
 def coreos_get_scmbasepath(d):
-    return os.path.normpath(os.path.join(d.getVar('COREBASE'), '../..'))
+    return d.getVar('COREOS_ROOT')
 
 def coreos_detect_revision(d):
     path = coreos_get_scmbasepath(d)
@@ -13,7 +15,5 @@ def coreos_detect_branch(d):
     path = coreos_get_scmbasepath(d)
     return base_get_metadata_git_branch(path, d)
 
-# metadata_scm.bbclass set this two variable to the branch and revision of 
+COREOS_METADATA_BRANCH := "${@coreos_detect_branch(d)}"
-# the openembedded-core layer
+COREOS_METADATA_REVISION := "${@coreos_detect_revision(d)}"
-METADATA_BRANCH := "${@coreos_detect_branch(d)}"
-METADATA_REVISION := "${@coreos_detect_revision(d)}"

layers/meta-belden-coreos/conf/distro/belden-coreos.conf

@@ -4,7 +4,7 @@ MAINTAINER = "Belden CoreOS Team"
 
 INHERIT += "coreos_metadata_scm"
 
-DISTRO_VERSION = "0.0.1-${METADATA_BRANCH}+${METADATA_REVISION}"
+DISTRO_VERSION = "0.0.1-${COREOS_METADATA_BRANCH}+${COREOS_METADATA_REVISION}"
 DISTRO_CODENAME = "kirkstone"
 
 # Distro features and policies

layers/meta-belden-coreos/conf/layer.conf

@@ -12,6 +12,9 @@ BBFILE_PRIORITY_meta-belden-coreos = "6"
 LAYERDEPENDS_meta-belden-coreos = "core"
 LAYERSERIES_COMPAT_meta-belden-coreos = "kirkstone"
 
+# Set a variable to get to the top of the metadata location
+COREOS_ROOT = '${@os.path.normpath("${LAYERDIR}/../../")}'
+
 # Sanity Checks
 # ==============================================================================
 

layers/meta-belden-coreos/recipes-devtool/core-os/coreos-doc.bb

@@ -6,7 +6,7 @@ PV = "0.0.1+git${METADATA_REVISION}"
 inherit externalsrc
 inherit allarch
 
-EXTERNALSRC := "${THISDIR}/../../../../documentation"
+EXTERNALSRC := "${COREOS_ROOT}/documentation"
 DEPENDS += "python3-sphinx python3-sphinx-rtd-theme"
 
 do_compile() {

scripts/coreos-bblayers-envsub

@@ -0,0 +1,23 @@
+#!/bin/env bash
+
+usage()
+{
+  echo "Usage: coreos-bblayers-envsub key value"
+  echo "Replace ##key## with value inside ${BBLAYERSCONF}"
+  exit 0
+}
+
+if [ -z "${BUILDDIR}" ]; then
+    echo "BUILDDIR not set. Are you in a bitbake environment?"
+    exit 2
+fi
+
+BBLAYERSCONF="${BUILDDIR}/conf/bblayers.conf"
+
+if [[ $1 =~ ^(-h|--help)$ ]]; then
+    usage
+fi
+
+# Put the absolute path to the layers in bblayers.conf
+# Replace ##ARG1## with ARG2
+sed -i -e "s|##$1##|$2|g" "$BBLAYERSCONF"

templates/bblayers.conf.sample

@@ -7,6 +7,6 @@ BBFILES ?= ""
 
 BBLAYERS ?= " \
   ##OEROOT##/meta \
-  ##OEROOT##/../meta-belden-coreos \
+  ##COREOS_LAYERSDIR##/meta-belden-coreos \
-  ##OEROOT##/../meta-belden-marvell-bsp \
+  ##COREOS_LAYERSDIR##/meta-belden-marvell-bsp \
   "