jobs: added first draft of NWL build pipeline
this commit holds additionally a common jenkins file and a file containing the targets (machines) to build. Latter is used for the drop-down parameter in the build job. Signed-off-by: Marc Mattmüller <marc.mattmueller@netmodule.com>
This commit is contained in:
Marc Mattmüller
parent
e15a4abbf3
commit
5b22271083
|
|
@ -0,0 +1,127 @@
|
|||
// Loading code requires a NODE context
|
||||
// But we want the code accessible outside the node Context
|
||||
// So declare common (object created by the LOAD operation) outside the Node block.
|
||||
def common
|
||||
|
||||
// Preloaded the list of supported targets used for the choice parameters TARGET:
|
||||
def targetList
|
||||
|
||||
// This step is necessary to get the files directly from the git repo, as for the
|
||||
// first build it is not yet cloned into the workspace. Normally we use an agent
|
||||
// of a certain label (like core-os_buildagent) to have define credentials to get
|
||||
// the files. In the current case we use any agent.
|
||||
node() {
|
||||
// get the list of the targets:
|
||||
targetList = sh(returnStdout: true, script: "cat ./jobs/nwlTargets").trim()
|
||||
|
||||
// load common file
|
||||
common = load "./jobs/Jenkinsfile_Common"
|
||||
}
|
||||
|
||||
// declarative pipeline
|
||||
pipeline {
|
||||
agent any
|
||||
|
||||
parameters {
|
||||
choice(name: 'TARGET', choices: "${targetList}", description: 'choose the build target')
|
||||
string(name: 'BUILD_BRANCH', defaultValue: 'main', description: 'Enter the branch of the NWL to build (default = main), will skip deployment if not main')
|
||||
booleanParam(name: 'CLEAN_BUILD', defaultValue: false, description: 'do a clean build, i.e. remove the yocto directory and start from scratch')
|
||||
booleanParam(name: 'DEBUGGING', defaultValue: false, description: 'debugging mode, removes quiet mode for bitbake')
|
||||
}
|
||||
|
||||
options {
|
||||
timeout(time: 5, unit: 'HOURS')
|
||||
disableConcurrentBuilds()
|
||||
buildDiscarder(
|
||||
logRotator(numToKeepStr: '20', daysToKeepStr: '7')
|
||||
)
|
||||
}
|
||||
|
||||
triggers {
|
||||
cron('H H(5-6) * * 1-5')
|
||||
}
|
||||
|
||||
stages {
|
||||
|
||||
stage('Check Parameters') {
|
||||
steps {
|
||||
script {
|
||||
printJobParameters()
|
||||
checkJobParameters()
|
||||
setDisplayName()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Prepare') {
|
||||
steps {
|
||||
script {
|
||||
if(params.CLEAN_BUILD) {
|
||||
println "CLEAN BUILD REQUESTED, cleaning..."
|
||||
common.cleaningClonedRepoDir()
|
||||
}
|
||||
setupEnvironment(common)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Build') {
|
||||
steps {
|
||||
script {
|
||||
dir("${env.YOCTO_REPO_DIR}") {
|
||||
common.buildTheYoctoPackage()
|
||||
def artifactName = "NWL-${machine}.zip"
|
||||
common.collectingPackageArtifacts("${env.MACHINE}")
|
||||
common.packAndArchiveArtifacts("${env.MACHINE}", artifactName)
|
||||
}
|
||||
println "TODO: sync sstate-cache to the server"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Deploy') {
|
||||
when { expression { return common.isCurrentJobSuccess() } }
|
||||
steps {
|
||||
script {
|
||||
println "TODO: Deploy artifacts"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // stages
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
def printJobParameters() {
|
||||
println "----------------------------------\n\
|
||||
Job Parameters:\n\
|
||||
----------------------------------\n\
|
||||
TARGET = ${params.TARGET}\n\
|
||||
BUILD_BRANCH = ${params.BUILD_BRANCH}\n\
|
||||
CLEAN_BUILD = ${params.CLEAN_BUILD}\n\
|
||||
DEBUGGING = ${params.DEBUGGING}\n\
|
||||
----------------------------------\n"
|
||||
}
|
||||
//---------------------------------------------------------------------------------------------------------------------
|
||||
def checkJobParameters() {
|
||||
println "ToDo: Check if triggered by timer before throwing error"
|
||||
if("${params.TARGET}" == "select...") {
|
||||
currentBuild.result = 'ABORTED'
|
||||
error("Missing build target --> select parameter TARGET for a proper build")
|
||||
}
|
||||
}
|
||||
//---------------------------------------------------------------------------------------------------------------------
|
||||
def setDisplayName() {
|
||||
def buildName = "#${env.BUILD_NUMBER}"
|
||||
currentBuild.displayName = "${buildName}"-${params.TARGET}""
|
||||
}
|
||||
|
||||
|
||||
//---------------------------------------------------------------------------------------------------------------------
|
||||
def setupEnvironment(commonHelpers) {
|
||||
def machine = "${params.TARGET}"
|
||||
def nwlBranch = "${params.BUILD_BRANCH}"
|
||||
def nwlRepoDir = "${env.YOCTO_REPO_DIR}"
|
||||
commonHelpers.setupBuildEnvironment(machine, nwlBranch, nwlRepoDir, params.DEBUGGING)
|
||||
commonHelpers.printEnvironmentParameters()
|
||||
}
|
||||
|
|
@ -0,0 +1,316 @@
|
|||
//=============================================
|
||||
// NetModule Wireless Linux CI commons
|
||||
//=============================================
|
||||
echo "loading NWL CI common module..."
|
||||
|
||||
|
||||
// URLs
|
||||
//----------------------------
|
||||
env.BITBUCKET_LOCAL = "bitbucket.gad.local"
|
||||
env.BITBUCKET_URL = "https://${env.BITBUCKET_LOCAL}"
|
||||
|
||||
env.YOCTO_REPO_URL = "ssh://git@${env.BITBUCKET_LOCAL}:7999/nm-nsp/netmodule-wireless-linux.git"
|
||||
|
||||
env.STORAGE_URL = "http://nmrepo.netmodule.intranet"
|
||||
env.SSTATE_STORAGE_URL = "${env.STORAGE_URL}/core-os-sstate"
|
||||
|
||||
env.HASHSERVER = "172.16.70.254:8686"
|
||||
|
||||
// Yocto build definitions
|
||||
//----------------------------
|
||||
env.YOCTO_REPO_DIR = "nwl"
|
||||
env.YOCTO_RELEASE = 'kirkstone'
|
||||
|
||||
env.CI_IMAGE = "nwl-image"
|
||||
|
||||
|
||||
// Methods declared in external code are accessible
|
||||
// directly from other code in the external file
|
||||
// indirectly via the object created by the load operation
|
||||
// eg. extcode.build(...)
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
def isCurrentJobSuccess() {
|
||||
return (currentBuild.currentResult == 'SUCCESS')
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def cleaningClonedRepoDir() {
|
||||
println "cleaning the entire repository..."
|
||||
sh("git clean -ffdx")
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
def setupGlobalEnvironmentVariables(repoDir, machine) {
|
||||
env.MACHINE = "${machine}"
|
||||
env.WORK_DIR = "${WORKSPACE}/${repoDir}"
|
||||
env.SHARED_BUILD = "${env.WORK_DIR}/build"
|
||||
env.BUILD_DEPLOY_DIR = "${env.SHARED_BUILD}/tmp/deploy"
|
||||
env.IMG_DEPLOY_DIR = "${env.BUILD_DEPLOY_DIR}/images"
|
||||
env.LICENSE_DEPLOY_DIR = "${env.BUILD_DEPLOY_DIR}/licenses"
|
||||
env.SDK_DEPLOY_DIR = "${env.BUILD_DEPLOY_DIR}/sdk"
|
||||
env.BUILD_HISTORY_DIR = "${env.SHARED_BUILD}/buildhistory"
|
||||
env.SSTATE_CACHE = "${env.SHARED_BUILD}/sstate-cache"
|
||||
env.PKG_CONTENT_DIR = "${env.WORK_DIR}/tmp/build-output"
|
||||
env.DEPLOY_CONTENT_DIR = "${env.WORK_DIR}/toDeploy"
|
||||
env.DOWNLOAD_DIR = "${JENKINS_HOME}/downloads"
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def getBitbakePackage(machine) {
|
||||
// ToDo: handle here bitbake packages if they differ and depend on the machine
|
||||
return "${env.CI_IMAGE}"
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def removePreExistingYoctoConfigs(confPath) {
|
||||
if(fileExists("${env.YOCTO_REPO_DIR}/${confPath}")) {
|
||||
println "Removing the bitbake config to integrate new meta layers..."
|
||||
sh(script: "rm -rf ${env.YOCTO_REPO_DIR}/${confPath}")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
def gitCheckout(gitUrl, branchTag, repoDir, hasSubmodules) {
|
||||
println "checking out ${gitUrl} to ${repoDir}..."
|
||||
if(!fileExists("./${repoDir}")) {
|
||||
sshagent (credentials: [env.SSH_ID]) {
|
||||
def inclSubmodulesOpt = hasSubmodules ? "--recurse-submodules" : ""
|
||||
sh(script: "git clone ${inclSubmodulesOpt} ${gitUrl} ${repoDir}")
|
||||
}
|
||||
}
|
||||
dir("${repoDir}") {
|
||||
def updateSubmodulesCmd = hasSubmodules ? " && git submodule update --init --recursive" : ""
|
||||
sshagent (credentials: [env.SSH_ID]) {
|
||||
sh(script: "git fetch -ap && git fetch -t")
|
||||
sh(script: "git checkout ${branchTag} && git pull --rebase ${updateSubmodulesCmd}")
|
||||
}
|
||||
if(hasSubmodules) {
|
||||
submoduleStatus = sh(script: "git submodule status", returnStdout: true)
|
||||
println "${submoduleStatus}"
|
||||
}
|
||||
gitHistory = sh(returnStdout: true, script: "git log --pretty=oneline -3")
|
||||
println "Last 3 git commits:\n-----------------------------\n${gitHistory}"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
def getMachineNameConfig() {
|
||||
return "MACHINE ?= \"${env.MACHINE}\""
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def getDownloadDirConfig() {
|
||||
return "DL_DIR = \"${env.DOWNLOAD_DIR}\""
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def getSstateMirrorConfig() {
|
||||
def mirrorCfg = "SSTATE_MIRRORS = \"file://.* ${env.SSTATE_STORAGE_URL}/PATH\""
|
||||
def signatureHdl = "BB_SIGNATURE_HANDLER = \"OEEquivHash\""
|
||||
def hashSrv = "BB_HASHSERVE = \"${env.HASHSERVER}\""
|
||||
return "${signatureHdl}\n${hashSrv}\n${mirrorCfg}"
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def getArtifactConfig() {
|
||||
return "NWL_IMAGE_EXTRACLASSES += \"nwl-image-ci\""
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def setupConfigFile(confPath, confFile) {
|
||||
// Keep in mind: order of configurations: site.conf, auto.conf, local.conf
|
||||
dir("${env.YOCTO_REPO_DIR}") {
|
||||
def machineCfg = getMachineNameConfig()
|
||||
def downloadCfg = getDownloadDirConfig()
|
||||
def sstateCfg = getSstateMirrorConfig()
|
||||
def artifactCfg = getArtifactConfig()
|
||||
def autoCfg = "${machineCfg}\n${downloadCfg}\n${sstateCfg}\n${parallelCfg}\n${artifactCfg}\n"
|
||||
|
||||
if(!fileExists("./${confPath}")) {
|
||||
def sourceCmd = "source ${env.YOCTO_ENV}"
|
||||
println "Initial build detected, sourcing environment to create structures and files..."
|
||||
def srcEnvStatus = sh(returnStatus: true, script: "bash -c '${sourceCmd} > /dev/null 2>&1'")
|
||||
println " -> status sourcing the yocto env = ${srcEnvStatus}"
|
||||
}
|
||||
writeFile(file: "${confFile}", text: "${autoCfg}")
|
||||
}
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def setupEnvironmentForArtifacts(machine) {
|
||||
// NOTE: this part depends on
|
||||
// - the path defined in env.YOCTO_DEPLOYS
|
||||
// - the target as defined in env.BITBAKE_PKG
|
||||
// - the yocto config preparation as done in setupConfigFile()
|
||||
// - the specific configuration as stated in getArtifactConfig()
|
||||
// and affects the function getPackageArtifacts()
|
||||
env.YOCTO_ARTIFACTS = "${env.YOCTO_DEPLOYS}/${env.BITBAKE_PKG}-${machine}.ci-artifacts"
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def setupBuildEnvironment(machine, branchTag, cloneDir, isDebug) {
|
||||
// with the machine parameter it will be possible to set up different
|
||||
// environment variables in here. Currently we use the SolidRun board
|
||||
|
||||
setupGlobalEnvironmentVariables(cloneDir, machine)
|
||||
|
||||
def confPath = "build/conf"
|
||||
env.RELATIVE_AUTOCONF_FILE = "${confPath}/auto.conf"
|
||||
|
||||
env.YOCTO_DEPLOYS = "${env.IMG_DEPLOY_DIR}/${machine}"
|
||||
|
||||
env.YOCTO_ENV = "nwl-init-build-env"
|
||||
env.BITBAKE_PKG = getBitbakePackage(machine)
|
||||
env.ISQUIET = isDebug.toBoolean() ? "" : "-q"
|
||||
env.BITBAKE_CMD = "${env.ISQUIET} ${env.BITBAKE_PKG}"
|
||||
|
||||
removePreExistingYoctoConfigs(confPath)
|
||||
gitCheckout("${env.YOCTO_REPO_URL}", branchTag, cloneDir, true)
|
||||
|
||||
env.PKG_NAME = "${env.BITBAKE_PKG}-${machine}"
|
||||
sh("mkdir -p ${env.DEPLOY_CONTENT_DIR}")
|
||||
|
||||
setupConfigFile(confPath, "${env.RELATIVE_AUTOCONF_FILE}")
|
||||
setupEnvironmentForArtifacts(machine)
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
def printEnvironmentParameters() {
|
||||
println "----------------------------------\n\
|
||||
Environment Parameters:\n\
|
||||
\n\
|
||||
--> machine = ${env.MACHINE}\n\
|
||||
--> git URL = ${env.YOCTO_REPO_URL}\n\
|
||||
--> yocto dir = ${env.YOCTO_REPO_DIR}\n\
|
||||
--> shared build dir = ${env.SHARED_BUILD}\n\
|
||||
--> autoconf file = ${env.RELATIVE_AUTOCONF_FILE}\n\
|
||||
--> yocto deploys = ${env.YOCTO_DEPLOYS}\n\
|
||||
--> yocto environment = ${env.YOCTO_ENV}\n\
|
||||
--> bitbake pagkage = ${env.BITBAKE_PKG}\n\
|
||||
--> pagkage name = ${env.PKG_NAME}\n\
|
||||
--> download dir = ${env.DOWNLOAD_DIR }\n\
|
||||
--> artifacts file = ${env.YOCTO_ARTIFACTS}\n\
|
||||
----------------------------------\n"
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// check Yocto output file for warnings and print them
|
||||
def checkAndHintOnWarnings(yoctoOutputFile) {
|
||||
def warnFindCmd = "cat \"${yoctoOutputFile}\" | grep \"WARNING:\" || true"
|
||||
def foundWarnings = sh(returnStdout: true, script: "${warnFindCmd}")
|
||||
if("${foundWarnings}" != "") {
|
||||
println "----------=< WARNINGS FOUND >=-----------\n${foundWarnings}\n-----------------------------------------\n"
|
||||
}
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
// retruns true if there is a fetch error
|
||||
def hasFetchError(yoctoOutputFile) {
|
||||
def hasFetchErrorCmd = "cat \"${yoctoOutputFile}\" | grep \"FetchError\" > /dev/null 2>&1 && exit 1 || exit 0"
|
||||
return (sh(returnStatus: true, script: "bash -c '${hasFetchErrorCmd}'") != 0).toBoolean()
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
// retruns true if bitbake is unable to connect
|
||||
def isBitbakeUnableToConnect(yoctoOutputFile) {
|
||||
def errMsg= "ERROR: Unable to connect to bitbake server"
|
||||
def isUnableCmd = "cat \"${yoctoOutputFile}\" | grep \"${errMsg}\" > /dev/null 2>&1 && exit 1 || exit 0"
|
||||
return (sh(returnStatus: true, script: "bash -c '${isUnableCmd}'") != 0).toBoolean()
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
// kill any residing bitbake processes on errors
|
||||
def killResidingBitbakeProcessesAtError(yoctoOutputFile) {
|
||||
if(hasFetchError(yoctoOutputFile) || isBitbakeUnableToConnect(yoctoOutputFile)) {
|
||||
println "Fetch- or connection error detected, killing residing bitbake processes..."
|
||||
def getBbPidCmd = "ps -ax | grep bitbake | grep -v grep | head -n 1 | sed -e 's/^[ \t]*//' | cut -d' ' -f1"
|
||||
def bitbakePid = sh(returnStdout: true, script: "${getBbPidCmd}")
|
||||
if("${bitbakePid}" != "") {
|
||||
println "Residing process found: ${bitbakePid}"
|
||||
sh("kill -9 ${bitbakePid}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
def buildTheYoctoPackage() {
|
||||
def yoctoOutFile = "yocto.out"
|
||||
def sourceCall = "source ${env.YOCTO_ENV} > ${env.SHARED_BUILD}/${yoctoOutFile} 2>&1"
|
||||
def buildCall = "bitbake ${env.BITBAKE_CMD} >> ${env.SHARED_BUILD}/${yoctoOutFile} 2>&1"
|
||||
def buildCmd = "${sourceCall}; ${buildCall}"
|
||||
def bitbakeStatus = 0;
|
||||
sshagent (credentials: [env.SSH_ID]) {
|
||||
bitbakeStatus = sh(returnStatus: true, script: "bash -c '${buildCmd}'")
|
||||
}
|
||||
println "bitbakeStatus=${bitbakeStatus}"
|
||||
|
||||
if(fileExists("${env.SHARED_BUILD}/${yoctoOutFile}")) {
|
||||
if((bitbakeStatus != 0) || ("${env.ISQUIET}" == "")) {
|
||||
println "Yoco Build Output: ----------------"
|
||||
sh "cat ${env.SHARED_BUILD}/${yoctoOutFile}"
|
||||
println "-----------------------------------"
|
||||
}
|
||||
checkAndHintOnWarnings("${env.SHARED_BUILD}/${yoctoOutFile}")
|
||||
killResidingBitbakeProcessesAtError("${env.SHARED_BUILD}/${yoctoOutFile}")
|
||||
}
|
||||
|
||||
// Do clean-up
|
||||
sh "rm -f ${env.SHARED_BUILD}/${yoctoOutFile}"
|
||||
sh(script: "git clean -f ${env.RELATIVE_AUTOCONF_FILE}")
|
||||
|
||||
if(bitbakeStatus != 0) {
|
||||
error("Build error, check yocto build output")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// copy the package- and license manifest into the current directory
|
||||
def getManifests(machine, artifactPath, targetBaseName) {
|
||||
def pkgArtifactName = "${env.BITBAKE_PKG}-${machine}"
|
||||
def pkgManifestFile = "${artifactPath}/${pkgArtifactName}.manifest"
|
||||
println "Copying Manifests...\n\
|
||||
--> artifactPath = ${artifactPath}\n\
|
||||
--> pkgArtifactName = ${pkgArtifactName}\n\
|
||||
--> pkgManifestFile = ${pkgManifestFile}\n\
|
||||
--> targetBaseName = ${targetBaseName}"
|
||||
sh(label: "Copy Package Manifest", script: "cp ${pkgManifestFile} ${targetBaseName}.manifest")
|
||||
sh(label: "Copy License Manifest", script: """
|
||||
LATEST_LICENSE_DIR=\$(ls -Artd ${env.LICENSE_DEPLOY_DIR}/${pkgArtifactName}* | tail -n 1)
|
||||
cp \$LATEST_LICENSE_DIR/license.manifest ${targetBaseName}_license.manifest
|
||||
""")
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
// copy the yocto artifacts into the current directory
|
||||
def getPackageArtifacts(machine, artifactPath, artifactListFile) {
|
||||
println "Getting package artifacts and copy them to current directory...\n\
|
||||
--> artifactPath = ${artifactPath}\n\
|
||||
--> artifactListFile = ${artifactListFile}"
|
||||
|
||||
sh(label: "Copy ${machine} Package Artifacts", script: """
|
||||
cat ${artifactListFile}
|
||||
cat ${artifactListFile} | xargs -I % sh -c 'cp ${artifactPath}/% .'
|
||||
""")
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
// copy the package artifacts to destination directory for packing
|
||||
def collectingPackageArtifacts(machine) {
|
||||
dir("${env.PKG_CONTENT_DIR}") {
|
||||
println "Collecting yocto package artifacts (machine = ${machine})..."
|
||||
|
||||
def artifactPath = "${env.IMG_DEPLOY_DIR}/${machine}"
|
||||
getManifests(machine, "${artifactPath}", "./${machine}")
|
||||
getPackageArtifacts(machine, artifactPath, "${env.YOCTO_ARTIFACTS}")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// pack and archive the artifacts
|
||||
def packAndArchiveArtifacts(machine, pkgArchiveName) {
|
||||
println "archiving the yocto package artifacts (machine = ${machine})..."
|
||||
dir ('tmp/artifacts') {
|
||||
zip archive: true, dir: "${env.PKG_CONTENT_DIR}", glob: "*", zipFile: "${pkgArchiveName}"
|
||||
sh("cp ${pkgArchiveName} ${env.DEPLOY_CONTENT_DIR}/")
|
||||
}
|
||||
sh("rm -rf ${env.PKG_CONTENT_DIR}/*")
|
||||
sh("rm -rf tmp/artifacts")
|
||||
}
|
||||
|
||||
|
||||
// !!Important Boilerplate!!
|
||||
// The external code must return it's contents as an object
|
||||
return this;
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
select...
|
||||
cn9130-cf-pro
|
||||
Loading…
Reference in New Issue