From e221f49cbf50aae44471b4f82766aa3f4121905b Mon Sep 17 00:00:00 2001 From: Samuel Dolt Date: Wed, 26 Oct 2022 14:09:18 +0200 Subject: [PATCH] feat: allow CoreOS to be embedded in another repository --- .vscode/extensions.json | 6 + coreos-init-build-env | 88 +++++-- documentation/conf.py | 3 +- documentation/index.rst | 1 + documentation/ref-manual/classes.rst | 12 +- documentation/ref-manual/index.rst | 1 + documentation/ref-manual/variables.rst | 31 +++ documentation/using-coreos.rst | 249 ++++++++++++++++++ .../classes/coreos_metadata_scm.bbclass | 12 +- .../conf/distro/belden-coreos.conf | 2 +- layers/meta-belden-coreos/conf/layer.conf | 3 + .../recipes-devtool/core-os/coreos-doc.bb | 2 +- scripts/coreos-bblayers-envsub | 23 ++ templates/bblayers.conf.sample | 4 +- 14 files changed, 391 insertions(+), 46 deletions(-) create mode 100644 .vscode/extensions.json create mode 100644 documentation/ref-manual/variables.rst create mode 100644 documentation/using-coreos.rst create mode 100755 scripts/coreos-bblayers-envsub diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..c39b0c8 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "ms-vscode.makefile-tools", + "timonwong.shellcheck" + ] +} \ No newline at end of file diff --git a/coreos-init-build-env b/coreos-init-build-env index a9e039c..e5db2f8 100755 --- a/coreos-init-build-env +++ b/coreos-init-build-env @@ -3,53 +3,83 @@ # This script is used to setup the OE Build Envrionment # Normally this is called as '. ./core-init-build-env ' +# Configuration variables +# ------------------------------------------------------------------------------ +# We only set default value, so all configuration variable can be overriden +# if already set when sourcing this script + # On some shell, we can get the path of this script when sources. Otherwise we # use the current directory as a fallback -if [ -n "$BASH_SOURCE" ]; then - CORE_OS_ROOT=$(dirname "$BASH_SOURCE") -elif [ -n "$ZSH_NAME" ]; then - CORE_OS_ROOT=$(dirname "$0") -else - CORE_OS_ROOT="$(pwd)" +if [ -z "$COREOS_ROOT" ]; then + if [ -n "$BASH_SOURCE" ]; then + COREOS_ROOT=$(dirname "$BASH_SOURCE") + elif [ -n "$ZSH_NAME" ]; then + COREOS_ROOT=$(dirname "$0") + else + COREOS_ROOT="$(pwd)" + fi fi # Get a non relative path to the root directory -CORE_OS_ROOT=$(readlink -f "${CORE_OS_ROOT}") +COREOS_ROOT=$(readlink -f "${COREOS_ROOT}") # Set the path to bitbake, openembedded-core and the template directory -BITBAKEDIR="${CORE_OS_ROOT}/bitbake" -OEROOT="${CORE_OS_ROOT}/layers/openembedded-core" -TEMPLATECONF="${CORE_OS_ROOT}/templates" +# All theses values can be overriden by the caller of coreos-init-build-env +BITBAKEDIR="${BITBAKEDIR:-${COREOS_ROOT}/bitbake}" +OEROOT="${OEROOT:-${COREOS_ROOT}/layers/openembedded-core}" +TEMPLATECONF="${TEMPLATECONF:-${COREOS_ROOT}/templates}" # Sanity checks +# ------------------------------------------------------------------------------ + +# BITBAKEDIR, OEROOT, TEMPLATECONF and COREOS_ROOT can be overridden by our user +# so let's check that they have valid value + +if [ ! -f "${COREOS_ROOT}/coreos-init-build-env" ]; then + echo "Error: COREOS_ROOT ($COREOS_ROOT) isn't valid" >&2 + echo "If you are using CoreOS directly, try using this script from CoreOS root directory." >&2 + echo "If you are embedding coreos-init-build-env in another script, set COREOS_ROOT correctly there." >&2 + return 1 +fi -# This check detect if CORE_OS_ROOT is valid, by checking if the templates -# directory exists. Usefull has we use $(pwd) as a fallback method on some shell if [ ! -d "$TEMPLATECONF" ]; then - echo "Error: $TEMPLATECONF doesn't exist!" >&2 - echo "Please run this script in oe-init-build-env's directory." >&2 + echo "Error: TEMPLATECONF (${TEMPLATECONF}) doesn't exist!" >&2 + echo "Please check your TEMPLATECONF configuration." >&2 return 1 fi -# This check detect if BITBAKEDIR exist. It's a simple way to check that we have -# fetched our git submodules -if [ ! -d "$BITBAKEDIR" ]; then - echo "Error: $BITBAKEDIR doesn't exist!" >&2 +if [ ! -f "${BITBAKEDIR}/bin/bitbake" ]; then + echo "Error: BITBAKEDIR (${BITBAKEDIR}) isn't valid!" >&2 echo "Please ensure all git submodule are fetched." >&2 + echo "And check your BITBAKEDIR configuration." >&2 return 1 fi +if [ ! -f "${OEROOT}/oe-init-build-env" ]; then + echo "Error: OEROOT (${OEROOT}) isn't valid!" >&2 + echo "Please ensure all git submodule are fetched." >&2 + echo "And check your OEROOT configuration." >&2 + return 1 +fi + +# Build environmnet setup +# ------------------------------------------------------------------------------ + # Call the oe-init-build-env scripts of openembedded-core -. "${OEROOT}/oe-init-build-env" "${1:-$CORE_OS_ROOT/build}" +. "${OEROOT}/oe-init-build-env" "${1:-$COREOS_ROOT/build}" -# Add the scripts directory of CoreOS to the path -# Make sure our paths are at the beginning of $PATH -for newpath in "${CORE_OS_ROOT}/scripts"; do - # Remove any existences of $newpath from $PATH - PATH=$(echo $PATH | sed -re "s#(^|:)$newpath(:|$)#\2#g;s#^:##") +# Add the first argument of the function to the path +coreos_path_add() { + # Make sure our paths are at the beginning of $PATH + # Remove any existences of $1 from $PATH + PATH=$(echo "$PATH" | sed -re "s#(^|:)$1(:|$)#\2#g;s#^:##") - # Add $newpath to $PATH - PATH="$newpath:$PATH" -done -unset newpath -export PATH + # Add $1 to the PATH + PATH="$1:$PATH" + export PATH +} + +coreos_path_add "${COREOS_ROOT}/scripts" + +# Add support for ##COREOS_LAYERSDIR## inside of bblayer template +coreos-bblayers-envsub COREOS_LAYERSDIR "${COREOS_ROOT}/layers" diff --git a/documentation/conf.py b/documentation/conf.py index 29b946e..3d55a21 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -40,6 +40,7 @@ version = release extensions = [ 'sphinx.ext.extlinks', 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', ] # external links and substitutions @@ -77,7 +78,7 @@ templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'oe-workdir', 'oe-logs'] # -- Options for HTML output ------------------------------------------------- diff --git a/documentation/index.rst b/documentation/index.rst index 782bcfc..f8bec68 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -22,6 +22,7 @@ same structures. :caption: Introduction and Overview Quick Build + Setting up a CoreOS based distro .. toctree:: :maxdepth: 1 diff --git a/documentation/ref-manual/classes.rst b/documentation/ref-manual/classes.rst index 13564de..9dc50a7 100644 --- a/documentation/ref-manual/classes.rst +++ b/documentation/ref-manual/classes.rst @@ -3,7 +3,8 @@ Classes ******* This chapter document the classes that are provided by Belden CoreOS. Classes -provided by OpenEmbedded-Core are documented in the Yocto Reference Manual. +provided by OpenEmbedded-Core are documented in the +:external:doc:`Yocto Reference Manual `. .. _ref-classes-coreos-metadata-scm: .. index:: coreos_metadata_scm.bbclass @@ -12,10 +13,9 @@ provided by OpenEmbedded-Core are documented in the Yocto Reference Manual. =============================== The ``coreos_metadata_scm`` class is used inside the CoreOS distribution -configuration file to change the value of ``METADATA_BRANCH`` and -``METADATA_REVISION`` to the current Git branch and revision of the main CoreOS -repository instead of the branch and revision of the OpenEmbedded-Core Layer, as -set by the :external:ref:`metadata_scm ` class. +configuration file to set the variables ``COREOS_METADATA_BRANCH`` and +``COREOS_METADATA_REVISION`` to the current Git branch and revision of the main +CoreOS repository. The ``coreos_metadata_scm`` is automatically inherited if ``DISTRO`` is set to ``belden-coreos`` or to any distro based on ``belden-coreos``. @@ -34,7 +34,7 @@ The ``coreos-image`` class provides common definitions for the .. index:: coreos-sanity.class ``coreos-sanity.bbclass`` -======================== +========================= The ``coreos-sanity`` class is inherited inside the CoreOS layer configuration file to add some sanity checks. Theses check ensure that the diff --git a/documentation/ref-manual/index.rst b/documentation/ref-manual/index.rst index 9edf112..c0c37f2 100644 --- a/documentation/ref-manual/index.rst +++ b/documentation/ref-manual/index.rst @@ -13,3 +13,4 @@ Belden CoreOS Reference Manual distro images features + variables diff --git a/documentation/ref-manual/variables.rst b/documentation/ref-manual/variables.rst new file mode 100644 index 0000000..d65d5ba --- /dev/null +++ b/documentation/ref-manual/variables.rst @@ -0,0 +1,31 @@ + +****************** +Variables Glossary +****************** + +This chapter lists common variables used in the CoreOS build +system and gives an overview of their function and contents. + +Variables provided by OpenEmbedded-Core are documented in the +:external:doc:`Yocto Reference Manual `. + + +.. glossary:: + :sorted: + + :term:`COREOS_ROOT` + + Specifies the root directory of CoreOS. + + It is an important distinction that :term:`COREOS_ROOT` points to root of + the Git repository of CoreOS, and not to a layer. + + :term:`COREOS_METADATA_BRANCH` + + The branch currently checked out for the CoreOS project (path + determined by :term:`COREOS_ROOT`). + + :term:`COREOS_METADATA_REVISION` + + The revision currently checked out for the CoreOS project (path + determined by :term:`COREOS_ROOT`). \ No newline at end of file diff --git a/documentation/using-coreos.rst b/documentation/using-coreos.rst new file mode 100644 index 0000000..27c24d2 --- /dev/null +++ b/documentation/using-coreos.rst @@ -0,0 +1,249 @@ +******************************** +Setting up a CoreOS based distro +******************************** + +This chapter explain how to setup a distro based in CoreOS + +Repository structures +##################### + +OpenEmbedded is a flexible tools, but we encourage each of our users to adopt +the same structure as CoreOS. In this chapter, replace each usage of `PRODUCT` +or `product` by a unique name related to your product. + +.. code-block:: + + product/ + ├── build/ (ignored by git) + ├── documentation/ + ├── layers/ + | └── coreos/ (submodule) + | | └── bitbake/ (submodule) + | | └── layers/ + | | | ├── openembedded-core (submodule) + | | | ├── meta-belden-coreos + | | | ├── meta-belden-coreos-bsp + | | | └── ... + | | └── ... + | ├── meta-product/ + | ├── meta-other-layers/ + | └── ... + ├── scripts/ + ├── templates/ + ├── product-init-build-env + ├── .gitignore + +Creating the structures +####################### + +.. code-block:: sh + + ~$ mkdir product + ~$ cd product + ~/product$ git init + ~/product$ git submodule init + + ~/product$ mkdir layers + ~/product$ mkdir scripts + + ~/product$ cd layers + ~/product/layers$ git clone --recurse-submodules ssh://git@bitbucket.gad.local:7999/ico/coreos.git + + ~/product/layers$ cd .. + ~/product$ cp -r layers/coreos/templates ./templates + ~/product$ cp layers/coreos/.gitignore ./.gitignore + + ~/product$ touch product-init-build-env + ~/product$ chmod +x product-init-build-env + ~/product$ nano product-init-build-env + +.. note:: + + By copying the .gitignore file of CoreOS, the build directory of the product + will not be stored inside of Git, which is the recommended approach as using + `devtool modify` modify the local `bblayers.conf`. Instead we recommend to + keep the template directory up to date so that a sane configuration can be + created when fetching the repository for the first time. + +Then you can enter the following inside the product-init-build-env files + +.. code-block:: sh + + #!/bin/sh + + # This script is used to setup the OE Build Envrionment + # Normally this is called as '. ./product-init-build-env ' + + # On some shell, we can get the path of this script when sources. Otherwise we + # use the current directory as a fallback + if [ -z "$PRODUCT_ROOT" ]; then + if [ -n "$BASH_SOURCE" ]; then + PRODUCT_ROOT=$(dirname "$BASH_SOURCE") + elif [ -n "$ZSH_NAME" ]; then + PRODUCT_ROOT=$(dirname "$0") + else + PRODUCT_ROOT="$(pwd)" + fi + fi + + # Get a non relative path to the root directory + PRODUCT_ROOT=$(readlink -f "${PRODUCT_ROOT}") + + # CoreOS init settings + COREOS_ROOT="${PRODUCT_ROOT}/layers/coreos" + TEMPLATECONF="${PRODUCT_ROOT}/templates" + + # Call the coreos-init-build-env scripts of CoreOS + . "${COREOS_ROOT}/coreos-init-build-env" "${1:-$PRODUCT_ROOT/build}" + + # From here the scripts and functions defined by CoreOS and + # OpenEmbedded-Core are available + + # Add support for ##PRODUCTS_LAYERSDIR## inside of bblayer template + coreos-bblayers-envsub PRODUCT_LAYERSDIR "${PRODUCT_ROOT}/layers" + + # Add the scripts directory of the product to the path + coreos_path_add "${PRODUCT_ROOT}/scripts" + +Using you new projects +###################### + +.. code-block:: sh + + ~product$ source product-init-build-env + +Creating your product layers +############################ + +You can create a new layers and add it to your active bblayers.conf file like +this: + +.. code-block:: sh + + ~product/build$ bitbake-layers create-layer ../layers/meta-belden-bsp + ~product/build$ bitbake-layers add-layer ../layers/meta-product + +Don't forget to update your templates `projects/templates/bblayers.conf.sample` +file. Inside this file use `##PRODUCT_LAYERSDIR##/meta-product` to have a +machine agnostic path. + +Optional: Change some git settings +################################## + +If you want to always `--recurse-submodules` when using `git pull`, you can +change your `submodule.recurse` git setting, either locally or globally + +.. code-block:: sh + + ~/product$ git config submodule.recurse true # Only inside of product repo + ~/product$ git config --global submodule.recurse true # Set it for all repos + + +Create your own distro based on CoreOS +###################################### + +Create a new file inside configuration file inside +`product/layers/meta-product/conf/distro`. For a distro named, you will create +`product/layers/meta-product/conf/distro/product.conf`. + +Open this file and enter the following: + +.. code-block:: ini + + # This should come at the beginning of the file, to ensure that you use + # CoreOS defaults + require conf/distro/belden-coreos.conf + + # This should always be set in your own configuration file, to not use the + # value of CoreOS + DISTRO = "product" + DISTRO_NAME = "Product Linux Distribution" + MAINTAINER = "Belden Product Team" + + # You may want to add a version and a codename to your distro instead of + # using the version and codename of CoreOS + DISTRO_VERSION = "2022.05" + DISTRO_CODENAME = "ProductOS Summer 2022 Edition" + + # From here you can overrides settings from the CoreOS distro or from + # OpenEmbedded-core. But keep in mind that the CoreOS teams doesn't support + # all the features of OpenEmbedded-Core. We have added some checks for some + # of the settings that we don't allow to change or that we don't support. + # See the coreos-sanity.bbclass file for more info. + +Then you can activate the distro by setting the `DISTRO` to `product` inside +your `product/build/conf/local.conf` file. You should also set it in the +`product/templates/local.conf.sample` file so that it will be set as the default +when create the build envrionment for the first time. + +What to do next +############### + +How do I... +----------- + +**...add a PRODUCT_ROOT variable usable in recipes files?** + +Add this line inside your meta-product layer configuration file at +`product/layers/meta-product/conf/layer.conf`: + +.. code-block:: ini + + # Set a variable to get to the top of the metadata location + PRODUCT_ROOT = '${@os.path.normpath("${LAYERDIR}/../../")}' + +**... add PRODUCT_METADATA_BRANCH and PRODUCT_METADATA_REVISION variables to get the +current git branch and git sha of the PRODUCT repository?** + +Create the file `product/layers/meta-product/classes/product_metadata_scm.bbclass` +and copy the content of the coreos_metadata_scm.bbclass file. Replacing all +reference to COREOS by PRODUCT should works. + +Alternative repository structure +################################ + +It's also possible but not recommended to clone CoreOS without any submodule, to +create a more flat structure. But then you have to ensure and manage the +Bitbake et OpenEmbedded-Core version by yourself. + +.. important:: + + CoreOS is only tested with the version of Bitbake and OpenEmbedded-Core used + in the CoreOS repository as submodule. By doing this you have to ensure that + you project stay in sync with CoreOS regarding CoreOS version and + corresponding Bitbake and OpenEmbedded-Core version. + + +.. code-block:: + + product/ + ├── build/ (ignored by git) + ├── bitbake/ (submodule) + ├── documentation/ + ├── layers/ + | ├── openembedded-core (submodule) + | └── coreos/ (cloned without submodule) + | | ├── layers/ + | | | ├── meta-belden-coreos + | | | ├── meta-belden-coreos-bsp + | | | └── ... + | | └── ... + | ├── meta-product/ + | ├── meta-other-layers/ + | └── ... + ├── scripts/ + ├── templates/ + ├── product-init-build-env + ├── .gitignore + +Setting this structure is out of the scope for this documentation, but as a +hint, to implement it you have to set in `product-init-build-env`: + +- `BITBAKEDIR` to the path of the Bitbake repository +- `OEROOT` to the path of the OpenEmbedded-Core repository + +.. important:: + + Calling directly oe-init-build-env from OpenEmbedded-Core is not supported! + Ensure that your product-init-build-env call coreos-init-build-env egal if + you use the recommended or alternative repository structures. diff --git a/layers/meta-belden-coreos/classes/coreos_metadata_scm.bbclass b/layers/meta-belden-coreos/classes/coreos_metadata_scm.bbclass index 280c121..64eebbf 100644 --- a/layers/meta-belden-coreos/classes/coreos_metadata_scm.bbclass +++ b/layers/meta-belden-coreos/classes/coreos_metadata_scm.bbclass @@ -1,9 +1,11 @@ -# This change the default value of variables defined into metadata_scm.bbclass +# This create the same variable for CoreOS as defined by metadata_scm.bbclass +# for OpenEmbedded-Core + # Note that metadata_scm.bbclass is already ihnerited by the base class, so we # don't need to add it here. def coreos_get_scmbasepath(d): - return os.path.normpath(os.path.join(d.getVar('COREBASE'), '../..')) + return d.getVar('COREOS_ROOT') def coreos_detect_revision(d): path = coreos_get_scmbasepath(d) @@ -13,7 +15,5 @@ def coreos_detect_branch(d): path = coreos_get_scmbasepath(d) return base_get_metadata_git_branch(path, d) -# metadata_scm.bbclass set this two variable to the branch and revision of -# the openembedded-core layer -METADATA_BRANCH := "${@coreos_detect_branch(d)}" -METADATA_REVISION := "${@coreos_detect_revision(d)}" +COREOS_METADATA_BRANCH := "${@coreos_detect_branch(d)}" +COREOS_METADATA_REVISION := "${@coreos_detect_revision(d)}" diff --git a/layers/meta-belden-coreos/conf/distro/belden-coreos.conf b/layers/meta-belden-coreos/conf/distro/belden-coreos.conf index a06e66e..69ca6f3 100644 --- a/layers/meta-belden-coreos/conf/distro/belden-coreos.conf +++ b/layers/meta-belden-coreos/conf/distro/belden-coreos.conf @@ -4,7 +4,7 @@ MAINTAINER = "Belden CoreOS Team" INHERIT += "coreos_metadata_scm" -DISTRO_VERSION = "0.0.1-${METADATA_BRANCH}+${METADATA_REVISION}" +DISTRO_VERSION = "0.0.1-${COREOS_METADATA_BRANCH}+${COREOS_METADATA_REVISION}" DISTRO_CODENAME = "kirkstone" # Distro features and policies diff --git a/layers/meta-belden-coreos/conf/layer.conf b/layers/meta-belden-coreos/conf/layer.conf index 5860c04..7f95f50 100644 --- a/layers/meta-belden-coreos/conf/layer.conf +++ b/layers/meta-belden-coreos/conf/layer.conf @@ -12,6 +12,9 @@ BBFILE_PRIORITY_meta-belden-coreos = "6" LAYERDEPENDS_meta-belden-coreos = "core" LAYERSERIES_COMPAT_meta-belden-coreos = "kirkstone" +# Set a variable to get to the top of the metadata location +COREOS_ROOT = '${@os.path.normpath("${LAYERDIR}/../../")}' + # Sanity Checks # ============================================================================== diff --git a/layers/meta-belden-coreos/recipes-devtool/core-os/coreos-doc.bb b/layers/meta-belden-coreos/recipes-devtool/core-os/coreos-doc.bb index 709a718..44cb05a 100644 --- a/layers/meta-belden-coreos/recipes-devtool/core-os/coreos-doc.bb +++ b/layers/meta-belden-coreos/recipes-devtool/core-os/coreos-doc.bb @@ -6,7 +6,7 @@ PV = "0.0.1+git${METADATA_REVISION}" inherit externalsrc inherit allarch -EXTERNALSRC := "${THISDIR}/../../../../documentation" +EXTERNALSRC := "${COREOS_ROOT}/documentation" DEPENDS += "python3-sphinx python3-sphinx-rtd-theme" do_compile() { diff --git a/scripts/coreos-bblayers-envsub b/scripts/coreos-bblayers-envsub new file mode 100755 index 0000000..c24474b --- /dev/null +++ b/scripts/coreos-bblayers-envsub @@ -0,0 +1,23 @@ +#!/bin/env bash + +usage() +{ + echo "Usage: coreos-bblayers-envsub key value" + echo "Replace ##key## with value inside ${BBLAYERSCONF}" + exit 0 +} + +if [ -z "${BUILDDIR}" ]; then + echo "BUILDDIR not set. Are you in a bitbake environment?" + exit 2 +fi + +BBLAYERSCONF="${BUILDDIR}/conf/bblayers.conf" + +if [[ $1 =~ ^(-h|--help)$ ]]; then + usage +fi + +# Put the absolute path to the layers in bblayers.conf +# Replace ##ARG1## with ARG2 +sed -i -e "s|##$1##|$2|g" "$BBLAYERSCONF" diff --git a/templates/bblayers.conf.sample b/templates/bblayers.conf.sample index ac5f1cf..ab16e37 100644 --- a/templates/bblayers.conf.sample +++ b/templates/bblayers.conf.sample @@ -7,6 +7,6 @@ BBFILES ?= "" BBLAYERS ?= " \ ##OEROOT##/meta \ - ##OEROOT##/../meta-belden-coreos \ - ##OEROOT##/../meta-belden-marvell-bsp \ + ##COREOS_LAYERSDIR##/meta-belden-coreos \ + ##COREOS_LAYERSDIR##/meta-belden-marvell-bsp \ "