From 246c859a35189a269536cc6c83470fc4c1ac57ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Tue, 29 Aug 2023 08:53:51 +0200 Subject: [PATCH 01/11] jobs/common: added cleanup function restoring unstaged changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit with the script updating the source revisions some unstaged changes reside in the repository. A following job will be unable to succeed as the clone will fail. Thus, adding this function which now can be used in the specific jobs. Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_Common | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jobs/Jenkinsfile_Common b/jobs/Jenkinsfile_Common index d4d5537..78441d4 100644 --- a/jobs/Jenkinsfile_Common +++ b/jobs/Jenkinsfile_Common @@ -144,6 +144,13 @@ def gitCheckout(gitUrl, branchTag, repoDir, hasSubmodules) { println "Last 3 git commits:\n-----------------------------\n${gitHistory}" } } +//----------------------------------------------------------------------------- +def cleanupRepository(repoDir) { + println "cleanup repository (e.g. temporary source revision updates)..." + dir("${repoDir}") { + sh("git restore .") + } +} //----------------------------------------------------------------------------- From 705aa2c1ade58ee21bf9b8f89c61ebf54f8608a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Tue, 29 Aug 2023 08:56:32 +0200 Subject: [PATCH 02/11] jobs/buildTarget: added post step cleaning repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the post step cleans the residuals from the update of the source revisions. Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_BuildTarget | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jobs/Jenkinsfile_BuildTarget b/jobs/Jenkinsfile_BuildTarget index 750771b..21a34d2 100644 --- a/jobs/Jenkinsfile_BuildTarget +++ b/jobs/Jenkinsfile_BuildTarget @@ -85,6 +85,14 @@ pipeline { } } } + post { + always { + script { + common.cleanupRepository("${env.YOCTO_REPO_DIR}") + } + } + } + } stage('Update SSTATE-CACHE MIRROR') { From a138da021ec8967bd07bf008c24d359bf382a527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Tue, 29 Aug 2023 09:01:16 +0200 Subject: [PATCH 03/11] jobs/buildAll: fixed build yocto job name and added clean-up of repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit with all the changes we have had regarding seed job integration in the HAC instance the job name changed. Thus, fixing the name when triggering the build job. Additionally the script updating the source revisions leaves some unstaged changes which is resolved by adding the clean-up of the repository after building all targets. Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_BuildAll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/Jenkinsfile_BuildAll b/jobs/Jenkinsfile_BuildAll index 771cd0c..3fd1d3d 100644 --- a/jobs/Jenkinsfile_BuildAll +++ b/jobs/Jenkinsfile_BuildAll @@ -116,7 +116,7 @@ def setupEnvironment(commonHelpers) { def nwlBranch = "${params.BUILD_BRANCH}" def nwlRepoDir = "${env.YOCTO_REPO_DIR}" commonHelpers.gitCheckout("${env.YOCTO_REPO_URL}", nwlBranch, nwlRepoDir, true) - env.BUILD_IMG_JOB = "/build-yocto-target-pipeline/${env.BRANCH_NAME}" + env.BUILD_IMG_JOB = "/nwl-target" } //----------------------------------------------------------------------------- @@ -183,7 +183,6 @@ def buildAllTargets(commonHelpers, buildBranch) { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { for (machine in listOfTargets) { - // ToDo: remove the if as soon as all machines are buildable if(isMachineSane(machine)) { Boolean isMachineSuccess = buildMachine(commonHelpers, machine, buildBranch) if(!isMachineSuccess && areBuildsSuccessful) { @@ -198,4 +197,5 @@ def buildAllTargets(commonHelpers, buildBranch) { error("Failing build - first machine failing = ${firstMachineFailing}") } } // end catchError + commonHelpers.cleanupRepository("${env.YOCTO_REPO_DIR}") } \ No newline at end of file From 54161e2b5c023ef093f478ece9f81e216a88de75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Tue, 29 Aug 2023 14:27:18 +0200 Subject: [PATCH 04/11] jobs: added seed for new job UpdateSrcRevisions (incl. job skeleton) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A request was to have the update source revision part in a separate job as in OEM Linux. This commit seeds a new job currently with an empty skeleton. Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_Seed | 35 ++++++++++++++++++++++++ jobs/Jenkinsfile_UpdateSrcRevisions | 41 +++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 jobs/Jenkinsfile_UpdateSrcRevisions diff --git a/jobs/Jenkinsfile_Seed b/jobs/Jenkinsfile_Seed index 45ec4e4..d1f3f48 100644 --- a/jobs/Jenkinsfile_Seed +++ b/jobs/Jenkinsfile_Seed @@ -65,3 +65,38 @@ pipelineJob('nwl-target') { } } } + + +pipelineJob('nwl-update-src-rev') { + displayName('0. Update Source Revisions') + description('CI pipeline updating the source revisions for the NetModule Wireless Linux') + authorization { + permissionAll('anonymous') + } + authenticationToken('buildNwlToken') + logRotator { + artifactDaysToKeep(7) + artifactNumToKeep(7) + daysToKeep(7) + numToKeep(7) + } + definition { + cpsScm { + scm { + git { + branches('main') + extensions { + cloneOptions { + depth(1) + } + } + remote { + credentials('nmgit_credentials') + url('ssh://git@bitbucket.gad.local:7999/nm-nsp/nwl-ci.git') + } + } + scriptPath('jobs/Jenkinsfile_UpdateSrcRevisions') + } + } + } +} diff --git a/jobs/Jenkinsfile_UpdateSrcRevisions b/jobs/Jenkinsfile_UpdateSrcRevisions new file mode 100644 index 0000000..000bbf9 --- /dev/null +++ b/jobs/Jenkinsfile_UpdateSrcRevisions @@ -0,0 +1,41 @@ +// Loading code requires a NODE context +// But we want the code accessible outside the node Context +// So declare common (object created by the LOAD operation) outside the Node block. +def common + +// declarative pipeline +pipeline { + agent any + + parameters { + string(name: 'BUILD_BRANCH', defaultValue: 'main', description: 'Enter the branch of the NWL to build (default = main), will skip deployment if not main') + booleanParam(name: 'CLEAN_BUILD', defaultValue: false, description: 'do a clean build, i.e. remove the yocto directory and start from scratch') + } + + options { + timeout(time: 1, unit: 'HOURS') + buildDiscarder( + logRotator(numToKeepStr: '10') + ) + } + + stages { + + stage('Prepare') { + steps { + script { + println "ToDo: prepare stage" + } + } + } + + stage('Update') { + steps { + script { + println "ToDo: Update stage" + } + } + } + + } // stages +} From be603724d26432b15f3502c920b394536b750e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Tue, 29 Aug 2023 14:40:31 +0200 Subject: [PATCH 05/11] jobs/common: refactored gitCheckout() for using more flexible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the function gitCheckout() holds the following steps: - git clone unless done - checking out a branch/tag/commit including rebase and updating submodules - launching the update source revision script - printing the submodule status - printing the last 3 commits of the git history But so far you could not use the steps separately. Thus, this function was refactored by extracting some steps into own functions so that they can be used easily. Example: the job workspace is the cloned git repository and you want to update the source revision hashes ;-P Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_Common | 50 ++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/jobs/Jenkinsfile_Common b/jobs/Jenkinsfile_Common index 78441d4..02892d9 100644 --- a/jobs/Jenkinsfile_Common +++ b/jobs/Jenkinsfile_Common @@ -118,8 +118,31 @@ def removePreExistingYoctoConfigs(confPath) { //----------------------------------------------------------------------------- -def gitCheckout(gitUrl, branchTag, repoDir, hasSubmodules) { - println "checking out ${gitUrl} to ${repoDir}..." +def printLatestGitHistory() { + gitHistory = sh(returnStdout: true, script: "git log --pretty=oneline -3") + println "Last 3 git commits:\n-----------------------------\n${gitHistory}" +} +//----------------------------------------------------------------------------- +def printSubmoduleStatus(hasSubmodules = true) { + if(hasSubmodules) { + submoduleStatus = sh(script: "git submodule status", returnStdout: true) + println "${submoduleStatus}" + } +} +//----------------------------------------------------------------------------- +def gitUpdateSubmodulesCheckoutBranch(branchTag, hasSubmodules, doUpdatePackageSrcRevisions) { + def gitCredentials = getGitCredentialID() + def updateSubmodulesCmd = hasSubmodules ? " && git submodule update --init --recursive" : "" + sshagent (credentials: [gitCredentials]) { + sh(script: "git checkout ${branchTag} && git pull --rebase ${updateSubmodulesCmd}") + if(doUpdatePackageSrcRevisions) { + sh(label: "Temporary task - updating source revisions (sub-repositories)", + script: "./scripts/update-source-revisions.sh") + } + } +} +//----------------------------------------------------------------------------- +def cloneGitRepoUnlessExisting(gitUrl, repoDir, hasSubmodules) { def gitCredentials = getGitCredentialID() if(!fileExists("./${repoDir}")) { sshagent (credentials: [gitCredentials]) { @@ -127,21 +150,16 @@ def gitCheckout(gitUrl, branchTag, repoDir, hasSubmodules) { sh(script: "git clone ${inclSubmodulesOpt} ${gitUrl} ${repoDir}") } } - dir("${repoDir}") { - def updateSubmodulesCmd = hasSubmodules ? " && git submodule update --init --recursive" : "" - sshagent (credentials: [gitCredentials]) { - sh(script: "git fetch -ap && git fetch -t") - sh(script: "git checkout ${branchTag} && git pull --rebase ${updateSubmodulesCmd}") +} +//----------------------------------------------------------------------------- +def gitCheckout(gitUrl, branchTag, repoDir, hasSubmodules, doUpdatePackageSrcRevisions = true) { + println "checking out git repository ${gitUrl} to ${repoDir}..." + cloneGitRepoUnlessExisting(gitUrl, repoDir, hasSubmodules) - sh(label: "Temporary task - updating source revisions (sub-repositories)", - script: "./scripts/update-source-revisions.sh") - } - if(hasSubmodules) { - submoduleStatus = sh(script: "git submodule status", returnStdout: true) - println "${submoduleStatus}" - } - gitHistory = sh(returnStdout: true, script: "git log --pretty=oneline -3") - println "Last 3 git commits:\n-----------------------------\n${gitHistory}" + dir("${repoDir}") { + gitUpdateSubmodulesCheckoutBranch(branchTag, hasSubmodules, doUpdatePackageSrcRevisions) + printSubmoduleStatus(hasSubmodules) + printLatestGitHistory() } } //----------------------------------------------------------------------------- From 1840ec74ead5694cf6cedb44efe04cf6ad303921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Mon, 4 Sep 2023 08:38:40 +0200 Subject: [PATCH 06/11] docker/nwl-cicd.yaml: disable SCM poll for seed job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit when seeding a branch other than the default one the poll detects a SCM change and triggers right after a new seed job which might revert the jobs seeded by the previous job run. With this behavior it won't be possible to seed a specific and run the intentionaly seeded jobs. Thus, this commit disables SCM poll. Signed-off-by: Marc Mattmüller --- docker/nwl-cicd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/nwl-cicd.yaml b/docker/nwl-cicd.yaml index a78b134..20e6dac 100644 --- a/docker/nwl-cicd.yaml +++ b/docker/nwl-cicd.yaml @@ -84,6 +84,7 @@ jobs: credentials('gitCredentials') } extensions { + disableRemotePoll() wipeWorkspace() } } From 6779ef5846dcbaa0cbf5dfda8a74fa8abf530611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Mon, 4 Sep 2023 09:07:34 +0200 Subject: [PATCH 07/11] jobs/seed: added nightly trigger with replaced pipelinetriggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit since job dsl plugin 1.77 the option 'triggers' is deprecated and will be removed (causes problems: https://issues.jenkins.io/browse/JENKINS-53775), see https://github.com/jenkinsci/job-dsl-plugin/wiki/Migration#migrating-to-177 Thus the option triggers was replaced Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_Seed | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jobs/Jenkinsfile_Seed b/jobs/Jenkinsfile_Seed index d1f3f48..c3bb7cf 100644 --- a/jobs/Jenkinsfile_Seed +++ b/jobs/Jenkinsfile_Seed @@ -11,6 +11,15 @@ pipelineJob('nwl') { daysToKeep(7) numToKeep(7) } + properties { + pipelineTriggers { + triggers { + cron { + spec('H H(5-6) * * 1-5') + } + } + } + } definition { cpsScm { scm { From a8e655a0d8142ea583791883f336842c5f2f5bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Mon, 4 Sep 2023 09:26:49 +0200 Subject: [PATCH 08/11] jobs/seed: use the parameter of the seeded branch for the seede jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The branch for the jobs to seed was set to main but we might want to seed the branch develop. Thus the parameter of the seed job replaces the hardcoded main. I am not sure if this solution works but wihtout trying we won't know :-D Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_Seed | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jobs/Jenkinsfile_Seed b/jobs/Jenkinsfile_Seed index c3bb7cf..9dcdf0a 100644 --- a/jobs/Jenkinsfile_Seed +++ b/jobs/Jenkinsfile_Seed @@ -24,7 +24,7 @@ pipelineJob('nwl') { cpsScm { scm { git { - branches('main') + branches('$SEEDING_BRANCH') extensions { cloneOptions { depth(1) @@ -58,7 +58,7 @@ pipelineJob('nwl-target') { cpsScm { scm { git { - branches('main') + branches('$SEEDING_BRANCH') extensions { cloneOptions { depth(1) @@ -93,7 +93,7 @@ pipelineJob('nwl-update-src-rev') { cpsScm { scm { git { - branches('main') + branches('$SEEDING_BRANCH') extensions { cloneOptions { depth(1) From bd63e877192dac6cc731476c7b249d3cce57bf7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Mon, 4 Sep 2023 09:41:58 +0200 Subject: [PATCH 09/11] jobs/seed: use variable for branch and use environment syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_Seed | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jobs/Jenkinsfile_Seed b/jobs/Jenkinsfile_Seed index 9dcdf0a..fd76921 100644 --- a/jobs/Jenkinsfile_Seed +++ b/jobs/Jenkinsfile_Seed @@ -1,3 +1,5 @@ +def branchToUse = "${env.SEEDING_BRANCH}" + pipelineJob('nwl') { displayName('1. NWL CI Pipeline') description('CI pipeline for the NetModule Wireless Linux') @@ -24,7 +26,7 @@ pipelineJob('nwl') { cpsScm { scm { git { - branches('$SEEDING_BRANCH') + branches(branchToUse) extensions { cloneOptions { depth(1) @@ -58,7 +60,7 @@ pipelineJob('nwl-target') { cpsScm { scm { git { - branches('$SEEDING_BRANCH') + branches(branchToUse) extensions { cloneOptions { depth(1) @@ -93,7 +95,7 @@ pipelineJob('nwl-update-src-rev') { cpsScm { scm { git { - branches('$SEEDING_BRANCH') + branches(branchToUse) extensions { cloneOptions { depth(1) From e0ae8a99e0cf4a1ccc094550d4f12ce3aaa7cb0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Mon, 4 Sep 2023 10:02:19 +0200 Subject: [PATCH 10/11] jobs/seed: moved environment getter into pipeline statement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_Seed | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jobs/Jenkinsfile_Seed b/jobs/Jenkinsfile_Seed index fd76921..9cfb6f6 100644 --- a/jobs/Jenkinsfile_Seed +++ b/jobs/Jenkinsfile_Seed @@ -1,6 +1,6 @@ -def branchToUse = "${env.SEEDING_BRANCH}" - pipelineJob('nwl') { + def branchToUse = binding.variables.get("SEEDING_BRANCH") + displayName('1. NWL CI Pipeline') description('CI pipeline for the NetModule Wireless Linux') authorization { @@ -44,6 +44,8 @@ pipelineJob('nwl') { } pipelineJob('nwl-target') { + def branchToUse = binding.variables.get("SEEDING_BRANCH") + displayName('2. NWL Yocto Build') description('CI pipeline building a NetModule Wireless Linux Yocto Target') authorization { @@ -79,6 +81,8 @@ pipelineJob('nwl-target') { pipelineJob('nwl-update-src-rev') { + def branchToUse = binding.variables.get("SEEDING_BRANCH") + displayName('0. Update Source Revisions') description('CI pipeline updating the source revisions for the NetModule Wireless Linux') authorization { From c79814899bd32b108d7ef308581bd4e688a1e160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Mattm=C3=BCller?= Date: Mon, 4 Sep 2023 10:25:29 +0200 Subject: [PATCH 11/11] jobs/common: update the yocto target (env.CI_IMAGE) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc Mattmüller --- jobs/Jenkinsfile_Common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/Jenkinsfile_Common b/jobs/Jenkinsfile_Common index 02892d9..4279033 100644 --- a/jobs/Jenkinsfile_Common +++ b/jobs/Jenkinsfile_Common @@ -27,7 +27,7 @@ env.HASHSERVER = "${env.HASH_SSTATE_SRV_IP}:8686" env.YOCTO_REPO_DIR = "nwl" env.YOCTO_RELEASE = 'kirkstone' -env.CI_IMAGE = "nwl-fitimage" +env.CI_IMAGE = "nwl-ramdisk-minimal" // Artifactory