Pull request #9: feat: allow CoreOS to be embedded in another repository

Merge in ICO/coreos from feat/embeding-coreos to master

* commit 'e221f49cbf50aae44471b4f82766aa3f4121905b':
  feat: allow CoreOS to be embedded in another repository
This commit is contained in:
Samuel Dolt 2022-10-27 20:02:37 +02:00
commit 5adcb7b476
14 changed files with 391 additions and 46 deletions

6
.vscode/extensions.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"recommendations": [
"ms-vscode.makefile-tools",
"timonwong.shellcheck"
]
}

View File

@ -3,53 +3,83 @@
# This script is used to setup the OE Build Envrionment
# Normally this is called as '. ./core-init-build-env <builddir>'
# Configuration variables
# ------------------------------------------------------------------------------
# We only set default value, so all configuration variable can be overriden
# if already set when sourcing this script
# On some shell, we can get the path of this script when sources. Otherwise we
# use the current directory as a fallback
if [ -n "$BASH_SOURCE" ]; then
CORE_OS_ROOT=$(dirname "$BASH_SOURCE")
elif [ -n "$ZSH_NAME" ]; then
CORE_OS_ROOT=$(dirname "$0")
else
CORE_OS_ROOT="$(pwd)"
if [ -z "$COREOS_ROOT" ]; then
if [ -n "$BASH_SOURCE" ]; then
COREOS_ROOT=$(dirname "$BASH_SOURCE")
elif [ -n "$ZSH_NAME" ]; then
COREOS_ROOT=$(dirname "$0")
else
COREOS_ROOT="$(pwd)"
fi
fi
# Get a non relative path to the root directory
CORE_OS_ROOT=$(readlink -f "${CORE_OS_ROOT}")
COREOS_ROOT=$(readlink -f "${COREOS_ROOT}")
# Set the path to bitbake, openembedded-core and the template directory
BITBAKEDIR="${CORE_OS_ROOT}/bitbake"
OEROOT="${CORE_OS_ROOT}/layers/openembedded-core"
TEMPLATECONF="${CORE_OS_ROOT}/templates"
# All theses values can be overriden by the caller of coreos-init-build-env
BITBAKEDIR="${BITBAKEDIR:-${COREOS_ROOT}/bitbake}"
OEROOT="${OEROOT:-${COREOS_ROOT}/layers/openembedded-core}"
TEMPLATECONF="${TEMPLATECONF:-${COREOS_ROOT}/templates}"
# Sanity checks
# ------------------------------------------------------------------------------
# BITBAKEDIR, OEROOT, TEMPLATECONF and COREOS_ROOT can be overridden by our user
# so let's check that they have valid value
if [ ! -f "${COREOS_ROOT}/coreos-init-build-env" ]; then
echo "Error: COREOS_ROOT ($COREOS_ROOT) isn't valid" >&2
echo "If you are using CoreOS directly, try using this script from CoreOS root directory." >&2
echo "If you are embedding coreos-init-build-env in another script, set COREOS_ROOT correctly there." >&2
return 1
fi
# This check detect if CORE_OS_ROOT is valid, by checking if the templates
# directory exists. Usefull has we use $(pwd) as a fallback method on some shell
if [ ! -d "$TEMPLATECONF" ]; then
echo "Error: $TEMPLATECONF doesn't exist!" >&2
echo "Please run this script in oe-init-build-env's directory." >&2
echo "Error: TEMPLATECONF (${TEMPLATECONF}) doesn't exist!" >&2
echo "Please check your TEMPLATECONF configuration." >&2
return 1
fi
# This check detect if BITBAKEDIR exist. It's a simple way to check that we have
# fetched our git submodules
if [ ! -d "$BITBAKEDIR" ]; then
echo "Error: $BITBAKEDIR doesn't exist!" >&2
if [ ! -f "${BITBAKEDIR}/bin/bitbake" ]; then
echo "Error: BITBAKEDIR (${BITBAKEDIR}) isn't valid!" >&2
echo "Please ensure all git submodule are fetched." >&2
echo "And check your BITBAKEDIR configuration." >&2
return 1
fi
if [ ! -f "${OEROOT}/oe-init-build-env" ]; then
echo "Error: OEROOT (${OEROOT}) isn't valid!" >&2
echo "Please ensure all git submodule are fetched." >&2
echo "And check your OEROOT configuration." >&2
return 1
fi
# Build environmnet setup
# ------------------------------------------------------------------------------
# Call the oe-init-build-env scripts of openembedded-core
. "${OEROOT}/oe-init-build-env" "${1:-$CORE_OS_ROOT/build}"
. "${OEROOT}/oe-init-build-env" "${1:-$COREOS_ROOT/build}"
# Add the scripts directory of CoreOS to the path
# Make sure our paths are at the beginning of $PATH
for newpath in "${CORE_OS_ROOT}/scripts"; do
# Remove any existences of $newpath from $PATH
PATH=$(echo $PATH | sed -re "s#(^|:)$newpath(:|$)#\2#g;s#^:##")
# Add the first argument of the function to the path
coreos_path_add() {
# Make sure our paths are at the beginning of $PATH
# Remove any existences of $1 from $PATH
PATH=$(echo "$PATH" | sed -re "s#(^|:)$1(:|$)#\2#g;s#^:##")
# Add $newpath to $PATH
PATH="$newpath:$PATH"
done
unset newpath
export PATH
# Add $1 to the PATH
PATH="$1:$PATH"
export PATH
}
coreos_path_add "${COREOS_ROOT}/scripts"
# Add support for ##COREOS_LAYERSDIR## inside of bblayer template
coreos-bblayers-envsub COREOS_LAYERSDIR "${COREOS_ROOT}/layers"

View File

@ -40,6 +40,7 @@ version = release
extensions = [
'sphinx.ext.extlinks',
'sphinx.ext.intersphinx',
'sphinx.ext.todo',
]
# external links and substitutions
@ -77,7 +78,7 @@ templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'oe-workdir', 'oe-logs']
# -- Options for HTML output -------------------------------------------------

View File

@ -22,6 +22,7 @@ same structures.
:caption: Introduction and Overview
Quick Build <quick-build>
Setting up a CoreOS based distro <using-coreos>
.. toctree::
:maxdepth: 1

View File

@ -3,7 +3,8 @@ Classes
*******
This chapter document the classes that are provided by Belden CoreOS. Classes
provided by OpenEmbedded-Core are documented in the Yocto Reference Manual.
provided by OpenEmbedded-Core are documented in the
:external:doc:`Yocto Reference Manual <ref-manual/classes>`.
.. _ref-classes-coreos-metadata-scm:
.. index:: coreos_metadata_scm.bbclass
@ -12,10 +13,9 @@ provided by OpenEmbedded-Core are documented in the Yocto Reference Manual.
===============================
The ``coreos_metadata_scm`` class is used inside the CoreOS distribution
configuration file to change the value of ``METADATA_BRANCH`` and
``METADATA_REVISION`` to the current Git branch and revision of the main CoreOS
repository instead of the branch and revision of the OpenEmbedded-Core Layer, as
set by the :external:ref:`metadata_scm <ref-classes-metadata_scm>` class.
configuration file to set the variables ``COREOS_METADATA_BRANCH`` and
``COREOS_METADATA_REVISION`` to the current Git branch and revision of the main
CoreOS repository.
The ``coreos_metadata_scm`` is automatically inherited if ``DISTRO`` is set to
``belden-coreos`` or to any distro based on ``belden-coreos``.
@ -34,7 +34,7 @@ The ``coreos-image`` class provides common definitions for the
.. index:: coreos-sanity.class
``coreos-sanity.bbclass``
========================
=========================
The ``coreos-sanity`` class is inherited inside the CoreOS layer
configuration file to add some sanity checks. Theses check ensure that the

View File

@ -13,3 +13,4 @@ Belden CoreOS Reference Manual
distro
images
features
variables

View File

@ -0,0 +1,31 @@
******************
Variables Glossary
******************
This chapter lists common variables used in the CoreOS build
system and gives an overview of their function and contents.
Variables provided by OpenEmbedded-Core are documented in the
:external:doc:`Yocto Reference Manual <ref-manual/variables>`.
.. glossary::
:sorted:
:term:`COREOS_ROOT`
Specifies the root directory of CoreOS.
It is an important distinction that :term:`COREOS_ROOT` points to root of
the Git repository of CoreOS, and not to a layer.
:term:`COREOS_METADATA_BRANCH`
The branch currently checked out for the CoreOS project (path
determined by :term:`COREOS_ROOT`).
:term:`COREOS_METADATA_REVISION`
The revision currently checked out for the CoreOS project (path
determined by :term:`COREOS_ROOT`).

View File

@ -0,0 +1,249 @@
********************************
Setting up a CoreOS based distro
********************************
This chapter explain how to setup a distro based in CoreOS
Repository structures
#####################
OpenEmbedded is a flexible tools, but we encourage each of our users to adopt
the same structure as CoreOS. In this chapter, replace each usage of `PRODUCT`
or `product` by a unique name related to your product.
.. code-block::
product/
├── build/ (ignored by git)
├── documentation/
├── layers/
| └── coreos/ (submodule)
| | └── bitbake/ (submodule)
| | └── layers/
| | | ├── openembedded-core (submodule)
| | | ├── meta-belden-coreos
| | | ├── meta-belden-coreos-bsp
| | | └── ...
| | └── ...
| ├── meta-product/
| ├── meta-other-layers/
| └── ...
├── scripts/
├── templates/
├── product-init-build-env
├── .gitignore
Creating the structures
#######################
.. code-block:: sh
~$ mkdir product
~$ cd product
~/product$ git init
~/product$ git submodule init
~/product$ mkdir layers
~/product$ mkdir scripts
~/product$ cd layers
~/product/layers$ git clone --recurse-submodules ssh://git@bitbucket.gad.local:7999/ico/coreos.git
~/product/layers$ cd ..
~/product$ cp -r layers/coreos/templates ./templates
~/product$ cp layers/coreos/.gitignore ./.gitignore
~/product$ touch product-init-build-env
~/product$ chmod +x product-init-build-env
~/product$ nano product-init-build-env
.. note::
By copying the .gitignore file of CoreOS, the build directory of the product
will not be stored inside of Git, which is the recommended approach as using
`devtool modify` modify the local `bblayers.conf`. Instead we recommend to
keep the template directory up to date so that a sane configuration can be
created when fetching the repository for the first time.
Then you can enter the following inside the product-init-build-env files
.. code-block:: sh
#!/bin/sh
# This script is used to setup the OE Build Envrionment
# Normally this is called as '. ./product-init-build-env <builddir>'
# On some shell, we can get the path of this script when sources. Otherwise we
# use the current directory as a fallback
if [ -z "$PRODUCT_ROOT" ]; then
if [ -n "$BASH_SOURCE" ]; then
PRODUCT_ROOT=$(dirname "$BASH_SOURCE")
elif [ -n "$ZSH_NAME" ]; then
PRODUCT_ROOT=$(dirname "$0")
else
PRODUCT_ROOT="$(pwd)"
fi
fi
# Get a non relative path to the root directory
PRODUCT_ROOT=$(readlink -f "${PRODUCT_ROOT}")
# CoreOS init settings
COREOS_ROOT="${PRODUCT_ROOT}/layers/coreos"
TEMPLATECONF="${PRODUCT_ROOT}/templates"
# Call the coreos-init-build-env scripts of CoreOS
. "${COREOS_ROOT}/coreos-init-build-env" "${1:-$PRODUCT_ROOT/build}"
# From here the scripts and functions defined by CoreOS and
# OpenEmbedded-Core are available
# Add support for ##PRODUCTS_LAYERSDIR## inside of bblayer template
coreos-bblayers-envsub PRODUCT_LAYERSDIR "${PRODUCT_ROOT}/layers"
# Add the scripts directory of the product to the path
coreos_path_add "${PRODUCT_ROOT}/scripts"
Using you new projects
######################
.. code-block:: sh
~product$ source product-init-build-env
Creating your product layers
############################
You can create a new layers and add it to your active bblayers.conf file like
this:
.. code-block:: sh
~product/build$ bitbake-layers create-layer ../layers/meta-belden-bsp
~product/build$ bitbake-layers add-layer ../layers/meta-product
Don't forget to update your templates `projects/templates/bblayers.conf.sample`
file. Inside this file use `##PRODUCT_LAYERSDIR##/meta-product` to have a
machine agnostic path.
Optional: Change some git settings
##################################
If you want to always `--recurse-submodules` when using `git pull`, you can
change your `submodule.recurse` git setting, either locally or globally
.. code-block:: sh
~/product$ git config submodule.recurse true # Only inside of product repo
~/product$ git config --global submodule.recurse true # Set it for all repos
Create your own distro based on CoreOS
######################################
Create a new file inside configuration file inside
`product/layers/meta-product/conf/distro`. For a distro named, you will create
`product/layers/meta-product/conf/distro/product.conf`.
Open this file and enter the following:
.. code-block:: ini
# This should come at the beginning of the file, to ensure that you use
# CoreOS defaults
require conf/distro/belden-coreos.conf
# This should always be set in your own configuration file, to not use the
# value of CoreOS
DISTRO = "product"
DISTRO_NAME = "Product Linux Distribution"
MAINTAINER = "Belden Product Team"
# You may want to add a version and a codename to your distro instead of
# using the version and codename of CoreOS
DISTRO_VERSION = "2022.05"
DISTRO_CODENAME = "ProductOS Summer 2022 Edition"
# From here you can overrides settings from the CoreOS distro or from
# OpenEmbedded-core. But keep in mind that the CoreOS teams doesn't support
# all the features of OpenEmbedded-Core. We have added some checks for some
# of the settings that we don't allow to change or that we don't support.
# See the coreos-sanity.bbclass file for more info.
Then you can activate the distro by setting the `DISTRO` to `product` inside
your `product/build/conf/local.conf` file. You should also set it in the
`product/templates/local.conf.sample` file so that it will be set as the default
when create the build envrionment for the first time.
What to do next
###############
How do I...
-----------
**...add a PRODUCT_ROOT variable usable in recipes files?**
Add this line inside your meta-product layer configuration file at
`product/layers/meta-product/conf/layer.conf`:
.. code-block:: ini
# Set a variable to get to the top of the metadata location
PRODUCT_ROOT = '${@os.path.normpath("${LAYERDIR}/../../")}'
**... add PRODUCT_METADATA_BRANCH and PRODUCT_METADATA_REVISION variables to get the
current git branch and git sha of the PRODUCT repository?**
Create the file `product/layers/meta-product/classes/product_metadata_scm.bbclass`
and copy the content of the coreos_metadata_scm.bbclass file. Replacing all
reference to COREOS by PRODUCT should works.
Alternative repository structure
################################
It's also possible but not recommended to clone CoreOS without any submodule, to
create a more flat structure. But then you have to ensure and manage the
Bitbake et OpenEmbedded-Core version by yourself.
.. important::
CoreOS is only tested with the version of Bitbake and OpenEmbedded-Core used
in the CoreOS repository as submodule. By doing this you have to ensure that
you project stay in sync with CoreOS regarding CoreOS version and
corresponding Bitbake and OpenEmbedded-Core version.
.. code-block::
product/
├── build/ (ignored by git)
├── bitbake/ (submodule)
├── documentation/
├── layers/
| ├── openembedded-core (submodule)
| └── coreos/ (cloned without submodule)
| | ├── layers/
| | | ├── meta-belden-coreos
| | | ├── meta-belden-coreos-bsp
| | | └── ...
| | └── ...
| ├── meta-product/
| ├── meta-other-layers/
| └── ...
├── scripts/
├── templates/
├── product-init-build-env
├── .gitignore
Setting this structure is out of the scope for this documentation, but as a
hint, to implement it you have to set in `product-init-build-env`:
- `BITBAKEDIR` to the path of the Bitbake repository
- `OEROOT` to the path of the OpenEmbedded-Core repository
.. important::
Calling directly oe-init-build-env from OpenEmbedded-Core is not supported!
Ensure that your product-init-build-env call coreos-init-build-env egal if
you use the recommended or alternative repository structures.

View File

@ -1,9 +1,11 @@
# This change the default value of variables defined into metadata_scm.bbclass
# This create the same variable for CoreOS as defined by metadata_scm.bbclass
# for OpenEmbedded-Core
# Note that metadata_scm.bbclass is already ihnerited by the base class, so we
# don't need to add it here.
def coreos_get_scmbasepath(d):
return os.path.normpath(os.path.join(d.getVar('COREBASE'), '../..'))
return d.getVar('COREOS_ROOT')
def coreos_detect_revision(d):
path = coreos_get_scmbasepath(d)
@ -13,7 +15,5 @@ def coreos_detect_branch(d):
path = coreos_get_scmbasepath(d)
return base_get_metadata_git_branch(path, d)
# metadata_scm.bbclass set this two variable to the branch and revision of
# the openembedded-core layer
METADATA_BRANCH := "${@coreos_detect_branch(d)}"
METADATA_REVISION := "${@coreos_detect_revision(d)}"
COREOS_METADATA_BRANCH := "${@coreos_detect_branch(d)}"
COREOS_METADATA_REVISION := "${@coreos_detect_revision(d)}"

View File

@ -4,7 +4,7 @@ MAINTAINER = "Belden CoreOS Team"
INHERIT += "coreos_metadata_scm"
DISTRO_VERSION = "0.0.1-${METADATA_BRANCH}+${METADATA_REVISION}"
DISTRO_VERSION = "0.0.1-${COREOS_METADATA_BRANCH}+${COREOS_METADATA_REVISION}"
DISTRO_CODENAME = "kirkstone"
# Distro features and policies

View File

@ -12,6 +12,9 @@ BBFILE_PRIORITY_meta-belden-coreos = "6"
LAYERDEPENDS_meta-belden-coreos = "core"
LAYERSERIES_COMPAT_meta-belden-coreos = "kirkstone"
# Set a variable to get to the top of the metadata location
COREOS_ROOT = '${@os.path.normpath("${LAYERDIR}/../../")}'
# Sanity Checks
# ==============================================================================

View File

@ -6,7 +6,7 @@ PV = "0.0.1+git${METADATA_REVISION}"
inherit externalsrc
inherit allarch
EXTERNALSRC := "${THISDIR}/../../../../documentation"
EXTERNALSRC := "${COREOS_ROOT}/documentation"
DEPENDS += "python3-sphinx python3-sphinx-rtd-theme"
do_compile() {

23
scripts/coreos-bblayers-envsub Executable file
View File

@ -0,0 +1,23 @@
#!/bin/env bash
usage()
{
echo "Usage: coreos-bblayers-envsub key value"
echo "Replace ##key## with value inside ${BBLAYERSCONF}"
exit 0
}
if [ -z "${BUILDDIR}" ]; then
echo "BUILDDIR not set. Are you in a bitbake environment?"
exit 2
fi
BBLAYERSCONF="${BUILDDIR}/conf/bblayers.conf"
if [[ $1 =~ ^(-h|--help)$ ]]; then
usage
fi
# Put the absolute path to the layers in bblayers.conf
# Replace ##ARG1## with ARG2
sed -i -e "s|##$1##|$2|g" "$BBLAYERSCONF"

View File

@ -7,6 +7,6 @@ BBFILES ?= ""
BBLAYERS ?= " \
##OEROOT##/meta \
##OEROOT##/../meta-belden-coreos \
##OEROOT##/../meta-belden-marvell-bsp \
##COREOS_LAYERSDIR##/meta-belden-coreos \
##COREOS_LAYERSDIR##/meta-belden-marvell-bsp \
"