diff --git a/doc/src/setup/nwl-ci.rst b/doc/src/setup/nwl-ci.rst index 01c6794..344dfe7 100644 --- a/doc/src/setup/nwl-ci.rst +++ b/doc/src/setup/nwl-ci.rst @@ -1929,6 +1929,256 @@ Some side information about the directory structure of the JENKINS_HOME. The tre +- workspace (working directory for the version control system) +Forward the Seeded Branch as well for the Pipelines +#################################################### +Currently the pipelines that are created by the seed job are taken from the branch main. This shall change as we have +the seed job parameter ``SEEDING_BRANCH``. Its value needs to be used within the pipelines to seed. Therefore the +parameter needs to bind into the seed file. The changes are seen in the following diff: + +.. code-block:: + + diff --git a/jobs/Jenkinsfile_Seed b/jobs/Jenkinsfile_Seed + index c3bb7cf..9cfb6f6 100644 + --- a/jobs/Jenkinsfile_Seed + +++ b/jobs/Jenkinsfile_Seed + @@ -1,4 +1,6 @@ + pipelineJob('nwl') { + + def branchToUse = binding.variables.get("SEEDING_BRANCH") + + + displayName('1. NWL CI Pipeline') + description('CI pipeline for the NetModule Wireless Linux') + authorization { + @@ -24,7 +26,7 @@ pipelineJob('nwl') { + cpsScm { + scm { + git { + - branches('main') + + branches(branchToUse) + extensions { + cloneOptions { + depth(1) + @@ -42,6 +44,8 @@ pipelineJob('nwl') { + } + + pipelineJob('nwl-target') { + + def branchToUse = binding.variables.get("SEEDING_BRANCH") + + + displayName('2. NWL Yocto Build') + description('CI pipeline building a NetModule Wireless Linux Yocto Target') + authorization { + @@ -58,7 +62,7 @@ pipelineJob('nwl-target') { + cpsScm { + scm { + git { + - branches('main') + + branches(branchToUse) + extensions { + cloneOptions { + depth(1) + @@ -77,6 +81,8 @@ pipelineJob('nwl-target') { + + + pipelineJob('nwl-update-src-rev') { + + def branchToUse = binding.variables.get("SEEDING_BRANCH") + + + displayName('0. Update Source Revisions') + description('CI pipeline updating the source revisions for the NetModule Wireless Linux') + authorization { + @@ -93,7 +99,7 @@ pipelineJob('nwl-update-src-rev') { + cpsScm { + scm { + git { + - branches('main') + + branches(branchToUse) + extensions { + cloneOptions { + depth(1) + + + +Extract Update Source Revision Part Into Job +############################################ +Preliminary Information +*********************** +As we have already seen in NetModule's OEM Linux, some recipes refer to a specific source revision of a repository which +is independent from the OEM Linux Yocto project. Now there are cases where the project moves forward but the recipe +holds still this specific source revision but should be at the latest state. Therefore a workflow was created using +devtool to check whether there are new revisions available. Now, it would be possible to update the recipe source +revision automatically by using these information. This is very helpful in nightly builds to check if head is still +working as expected. At a release we want to fix those revisions which is possible too by automatically commit the +revisions found. + +This behavior can be reflected with a Jenkins job. + +Seeding an additional Job +************************* +The following diff shows you the changes for seeding a new job (where the job is a skeleton). + +.. code-block:: + + diff --git a/jobs/Jenkinsfile_Seed b/jobs/Jenkinsfile_Seed + index 45ec4e4..d1f3f48 100644 + --- a/jobs/Jenkinsfile_Seed + +++ b/jobs/Jenkinsfile_Seed + @@ -65,3 +65,38 @@ pipelineJob('nwl-target') { + } + } + } + + + + + +pipelineJob('nwl-update-src-rev') { + + def branchToUse = binding.variables.get("SEEDING_BRANCH") + + + + displayName('0. Update Source Revisions') + + description('CI pipeline updating the source revisions for the NetModule Wireless Linux') + + authorization { + + permissionAll('anonymous') + + } + + authenticationToken('buildNwlToken') + + logRotator { + + artifactDaysToKeep(7) + + artifactNumToKeep(7) + + daysToKeep(7) + + numToKeep(7) + + } + + definition { + + cpsScm { + + scm { + + git { + + branches(branchToUse) + + extensions { + + cloneOptions { + + depth(1) + + } + + } + + remote { + + credentials('nmgit_credentials') + + url('ssh://git@bitbucket.gad.local:7999/nm-nsp/nwl-ci.git') + + } + + } + + scriptPath('jobs/Jenkinsfile_UpdateSrcRevisions') + + } + + } + + } + +} + diff --git a/jobs/Jenkinsfile_UpdateSrcRevisions b/jobs/Jenkinsfile_UpdateSrcRevisions + new file mode 100644 + index 0000000..000bbf9 + --- /dev/null + +++ b/jobs/Jenkinsfile_UpdateSrcRevisions + @@ -0,0 +1,41 @@ + +// Loading code requires a NODE context + +// But we want the code accessible outside the node Context + +// So declare common (object created by the LOAD operation) outside the Node block. + +def common + + + +// declarative pipeline + +pipeline { + + agent any + + + + parameters { + + string(name: 'BUILD_BRANCH', defaultValue: 'main', description: 'Enter the branch of the NWL to build (default = main), will skip deployment if not main') + + booleanParam(name: 'CLEAN_BUILD', defaultValue: false, description: 'do a clean build, i.e. remove the yocto directory and start from scratch') + + } + + + + options { + + timeout(time: 1, unit: 'HOURS') + + buildDiscarder( + + logRotator(numToKeepStr: '10') + + ) + + } + + + + stages { + + + + stage('Prepare') { + + steps { + + script { + + println "ToDo: prepare stage" + + } + + } + + } + + + + stage('Update') { + + steps { + + script { + + println "ToDo: Update stage" + + } + + } + + } + + + + } // stages + +} + +.. caution:: + With the current setup the job seems to be sed but another seeding is triggered due to SCM change. See the following + subsection about how to solve this issue. + +Adapt initial Seed Job not triggering on SCM change +=================================================== +The initial seed job within the Jenkins instance needs to have a ``disableRemotePoll()`` so that the seed job is not +triggered on SCM change: + +.. code-block:: + + diff --git a/jenkins-ci/jobs/seed.groovy b/jenkins-ci/jobs/seed.groovy + index 033eacf..277785c 100644 + --- a/jenkins-ci/jobs/seed.groovy + +++ b/jenkins-ci/jobs/seed.groovy + @@ -58,6 +58,7 @@ freeStyleJob('nwl-seed-job') { + credentials('admin_credentials') + } + extensions { + + disableRemotePoll() + wipeWorkspace() + } + } + + +With this change we create a new version of the NWL docker instance: + +.. code-block:: + + # on your local machine: + DOCKER_BUILDKIT=1 ./build.sh nwl 0.3.2 + docker save nwl-env-ci:latest | bzip2 | pv | ssh user@10.115.101.98 docker load + + # on the build server: + docker image tag nwl-env-ci:latest nwl-env-ci:0.3.2 + ./manage.sh --name=nwl_0_3_1 destroy + ./manage.sh --image=nwl-env-ci:0.3.2 --branch=main --name=nwl_0_3_2 --platform=nwl --config=/home/user/work/ci/config/config.xml --revision=0.3.2 --maintainer=TeamCHBE create + + +Nightly Trigger Integration +########################### +With the Job DSL plugin version 1.77 the option ``triggers`` is deprecated and will be removed soon (reason: it caused +problems, see `ticket `_). A replacement is available as stated in +the `migration note `_. According to this +note, the nightly trigger uses the ``pipelineTriggers`` as follows: + +.. code-block:: + + diff --git a/jobs/Jenkinsfile_Seed b/jobs/Jenkinsfile_Seed + index d1f3f48..c3bb7cf 100644 + --- a/jobs/Jenkinsfile_Seed + +++ b/jobs/Jenkinsfile_Seed + @@ -11,6 +11,15 @@ pipelineJob('nwl') { + daysToKeep(7) + numToKeep(7) + } + + properties { + + pipelineTriggers { + + triggers { + + cron { + + spec('H H(5-6) * * 1-5') + + } + + } + + } + + } + definition { + cpsScm { + scm { + + .. |nwlCiChain| image:: ./media/nwl-ci-jenkins-dashboard.png :width: 700px diff --git a/jobs/Jenkinsfile_UpdateSrcRevisions b/jobs/Jenkinsfile_UpdateSrcRevisions index 000bbf9..932bbdf 100644 --- a/jobs/Jenkinsfile_UpdateSrcRevisions +++ b/jobs/Jenkinsfile_UpdateSrcRevisions @@ -10,6 +10,7 @@ pipeline { parameters { string(name: 'BUILD_BRANCH', defaultValue: 'main', description: 'Enter the branch of the NWL to build (default = main), will skip deployment if not main') booleanParam(name: 'CLEAN_BUILD', defaultValue: false, description: 'do a clean build, i.e. remove the yocto directory and start from scratch') + booleanParam(name: 'DRY_RUN', defaultValue: true, description: 'do a dry run, i.e. without committing and pushing') } options { @@ -24,7 +25,13 @@ pipeline { stage('Prepare') { steps { script { - println "ToDo: prepare stage" + printJobParameters() + setDisplayName() + + // load common file + common = load "./jobs/Jenkinsfile_Common" + + handleCleanBuild(common) } } } @@ -32,10 +39,80 @@ pipeline { stage('Update') { steps { script { - println "ToDo: Update stage" + updateTheSourceRevisions(common, "${params.BUILD_BRANCH}") + commitAndPushTheChanges(common) } } + post { + always { + script { + common.cleanupRepository("./") + } + } + } + } } // stages } + +//----------------------------------------------------------------------------- +def printJobParameters() { + println "----------------------------------\n\ + Job Parameters:\n\ +----------------------------------\n\ + BUILD_BRANCH = ${params.BUILD_BRANCH}\n\ + CLEAN_BUILD = ${params.CLEAN_BUILD}\n\ + DRY_RUN = ${params.DRY_RUN}\n\ +----------------------------------\n" +} +//--------------------------------------------------------------------------------------------------------------------- +def setDisplayName() { + def buildName = "#${env.BUILD_NUMBER}" + def theBranch = "${params.BUILD_BRANCH}" + def cleaning = ("${params.CLEAN_BUILD}" == true) ? "-clean" : "" + currentBuild.displayName = "${buildName}-${theBranch}${cleaning}" +} +//--------------------------------------------------------------------------------------------------------------------- +def handleCleanBuild(commonHelpers) { + if(params.CLEAN_BUILD) { + println "CLEAN BUILD REQUESTED, cleaning..." + commonHelpers.cleaningClonedRepoDir() + } +} + + +//--------------------------------------------------------------------------------------------------------------------- +def updateTheSourceRevisions(commonHelpers, theBranch) { + commonHelpers.gitUpdateSubmodulesCheckoutBranch(theBranch, true, true) + commonHelpers.printSubmoduleStatus(true) +} +//--------------------------------------------------------------------------------------------------------------------- +def commitAndPushTheChanges(commonHelpers) { + def commitMsg = "srcrev: updated source revisions by Jenkins Job" + + if(sh(returnStdout: true, script: "git status | grep \"modified:\" | grep -v coreos | wc -l").toInteger() != 0) { + def itemFindCmd = "git status | grep \"modified:\" | grep -v coreos | cut -d':' -f2 | sed -e 's/^[ ]*//' | cut -d' ' -f1" + def changedItems = sh(returnStdout: true, script: "${itemFindCmd}") + sh(label: "Stage tracked and changed git files", returnStdout: true, script: "git add ${changedItems}") + + if(params.DRY_RUN) { + println "DRY RUN: commit message = '${commitMsg}'\n changes = ${changedItems}" + sh(script: """ + git reset HEAD * > /dev/null + git checkout * > /dev/null + """) + } + else { + println "ToDo: commit and push the changes" + //NOTE: The following code snippet may help... + //def theCredentials = getGitCredentialID() + //sshagent (credentials: [theCredentials]) { + // sh(label: "Commit and push changes", returnStdout: true, script: """ + // git commit -m "${commitMsg}" + // git push + // """) + //} + } + } +}