jenkinsfile: split ramdisk and mmc images into separate pipelines

Common parts of those pipelines are extracted into a common part.
Each pipeline loads this common part as module to get access to
its functions.
This is also the base for adding a job generating the SDK

BugzID: 73564

Signed-off-by: Marc Mattmueller <marc.mattmueller@netmodule.com>

Jenkinsfile_Common

@@ -0,0 +1,158 @@
+// common parts for yocto builds
+
+env.SHARED_BUILD = "${WORKSPACE}/build"
+env.BUILD_DEPLOY_DIR = "${env.SHARED_BUILD}/tmp/deploy/images"
+env.BUILD_LICENSE_DIR = "${env.SHARED_BUILD}/tmp/deploy/licenses"
+env.BUILD_HISTORY_DIR = "${env.SHARED_BUILD}/buildhistory"
+env.DISTRO_VERSION_PATHNAME = "${env.SHARED_BUILD}/conf/distro_version.inc"
+env.DOWNLOAD_DIR = "${env.SHARED_BUILD}/downloads"
+env.BINARY_STORAGE_URL = "http://nmrepo.netmodule.intranet/src/yocto-downloads"
+env.SUBMODULE_VERION_FILE = "submodule_revisions"
+env.DISTRO_VERSION_FILE = "distro_version.inc"
+env.AUTOREV_VERSION_FILE = "autorev_revisions.inc"
+env.PACKAGE_NAME = "nm-os"
+
+
+echo "loading common yocto build module..."
+
+
+// Methods declared in external code are accessible
+//   directly from other code in the external file
+//   indirectly via the object created by the load operation
+//   eg.  extcode.build(...)
+
+
+def isRelease(versionParam) {
+  if((versionParam == "") || (versionParam == "latest")) {
+    return false
+  }
+  return true
+}
+
+
+def handleSubmodules(isForcedToHead) {
+  println "init submodules..."
+  sh 'git submodule init'
+  if(isForcedToHead) {
+    println "setting submodule hashes to head..."
+    sh 'git submodule update --remote --rebase' // update all submodules to HEAD
+  }
+  else {
+    sh 'git submodule update' // set all submodules to freezed commit
+  }
+  submoduleStatus = sh(returnStdout: true, script: "git submodule status").trim()  // print submodule hashes to jenkins log
+  println "${submoduleStatus}"
+  writeFile(file: "${env.SUBMODULE_VERION_FILE}", text: "${submoduleStatus}")
+}
+
+
+def getTopUpstreamBuildNumber() {
+  // Iterating though all upstream jobs:
+  //   currentBuild.upstreamBuilds.each { item ->
+  //     echo "upstream build: ${item}"
+  //     def nbr = item.getNumber()
+  //     echo "nbr=${nbr}"
+  //   }
+  def upstreamJobList = currentBuild.upstreamBuilds
+  def nbrOfUpstreamJobs = upstreamJobList.size()
+  if (nbrOfUpstreamJobs == 0)
+    return 0;
+  def topJob = upstreamJobList[nbrOfUpstreamJobs-1]
+  println "Top upstream project: " + topJob.getFullDisplayName()
+  def topJobNbr = topJob.getNumber()
+  println "Top upstream job build Number = ${topJobNbr}"
+  return topJobNbr
+}
+
+def buildVersionString(imageType, actualBaseVersionString, versionParameter) {
+  // official release version
+  if(isRelease(versionParameter)) {
+    String newVersionStr = versionParameter 
+    return newVersionStr
+  }
+
+  // FCT release
+  if(imageType == 'fct') {
+    return actualBaseVersionString
+  }
+
+  // nightly/incremental release
+  def buildnbr = getTopUpstreamBuildNumber()
+  String nightlyPart = actualBaseVersionString + ".Test${buildnbr}"
+  return nightlyPart
+}
+
+def getVersionString(versionParam, imageType) {
+  sh 'git fetch -ap'
+  sh 'git fetch -t'
+  def gitCmd = "git describe --tags"
+
+  if(!isRelease(versionParam)) {
+    gitCmd = "${gitCmd} --dirty"
+  }
+  gitversion = sh(returnStdout: true, script: "${gitCmd}").trim()
+  String[] versionArr = "${gitversion}".split("-")
+
+  versionArr[0] = buildVersionString(imageType, versionArr[0], versionParam)
+  rlsVersion = versionArr.join("-")
+  return rlsVersion
+}
+
+
+def changeDistroVersion(versionString){
+  println "Set the distro version to ${versionString}..."
+  def versionTag = "DISTRO_VERSION = \"${versionString}\""
+  writeFile(file: "${env.DISTRO_VERSION_PATHNAME}", text: "${versionTag}")
+  sh(script: "cp ${env.DISTRO_VERSION_PATHNAME} ${env.DISTRO_VERSION_FILE}")
+}
+
+def cleanupDistroVersion() {
+  println "cleaning repository regarding distro version..."
+  sh(script:"git clean -f ${env.DISTRO_VERSION_PATHNAME}")
+}
+
+
+def archiveImages(imgageDir) {
+  dir ('tmp/artifacts') {
+    zip archive: true, dir: "${WORKSPACE}/${imgageDir}", glob: "*", zipFile: "${env.PACKAGE_NAME}-${env.BUILD_VERSION}-${params.MACHINE}-${params.IMAGE_TYPE}.zip"
+  }
+  sh "rm -rf ${WORKSPACE}/tmp"
+}
+
+
+def syncSources(src, dst) {
+  def hasSrcUrl = (src.contains("http"))
+  def from = src
+  def to = dst
+
+  // convert the URL into ssh syntax:
+  def url = (hasSrcUrl) ? src : dst
+  String[] repoParts = url.split("//")[1].split("/")
+  repoParts[0] = "build_user@" + repoParts[0] + ":/repo/repo"
+  sshSrc = repoParts.join("/")
+
+  if(hasSrcUrl) {
+    println "getting data from server..."
+    from = sshSrc
+  }
+  else {
+    println "putting data to server..."
+    to = sshSrc
+  }
+
+  sshagent (credentials: ['7767e711-08a4-4c71-b080-197253dd7392']) {
+    sh "rsync -auvz --ignore-existing -e \"ssh\" ${from}/* ${to}"
+  }
+}
+
+
+def getAutoRevHashes(envType) {
+  def env = "${envType}" == "" ? "" : "-${envType}"
+  def revs = sh(returnStdout: true, script: "bash -c \". ./env.image${env} > /dev/null && buildhistory-collect-srcrevs\"").trim()
+  return revs
+}
+
+
+// !!Important Boilerplate!!
+// The external code must return it's contents as an object
+return this;

Jenkinsfile_RamdiskImages

@@ -0,0 +1,152 @@
+// Loading code requires a NODE context
+// But we want the code accessible outside the node Context
+// So declare yoctocommon (object created by the LOAD operation) outside the Node block.
+def yoctocommon
+
+
+// declarative pipeline
+pipeline {
+  agent {
+    node {
+      label 'lxbuild3'
+    }
+  }
+
+  parameters {
+    choice(name: 'MACHINE', choices: ['select...', 'am335x-nrhw20', 'am335x-nmhw21', 'imx8-nmhw23', 'am335x-nmhw24', 'am335x-hw25', 'am335x-hw26'], description: 'choose target platform')
+    choice(name: 'IMAGE_TYPE', choices: ['lava', 'fct', 'minimal'], description: 'choose image type')
+    string(name: 'RLS_VERSION', defaultValue: '', description: 'Set the version to build and use committed submodules')
+    booleanParam(name: 'CLEAN_BUILD', defaultValue: false, description: 'clean all temp directories before build starts')
+    booleanParam(name: 'DO_UPDATE_TO_HEAD', defaultValue: false, description: 'update submodules to head')
+  }
+
+  environment {
+    IMG_OUTPUT_DIR = "tmp/build-output"
+  }
+
+  options {
+    timeout(time: 8, unit: 'HOURS')
+    buildDiscarder(
+      logRotator(numToKeepStr: '50',
+                 daysToKeepStr: '3',
+                 artifactNumToKeepStr: '50',
+                 artifactDaysToKeepStr: '3'
+      )
+    )
+    disableConcurrentBuilds()
+  }
+
+  stages {
+    stage('prepare') {
+      steps {
+        script {
+          if("${params.MACHINE}" == "select...") {
+            error("Missing machine type --> select parameter MACHINE for a proper build")
+          }
+
+          // load yocto common file
+          env.ROOTDIR = pwd()
+          yoctocommon = load "${env.ROOTDIR}/Jenkinsfile_Common"
+
+          // Prepare Build Environment
+          env.YOCTO_DEPLOYS = "${env.BUILD_DEPLOY_DIR}/${params.MACHINE}"
+          yoctocommon.handleSubmodules(params.DO_UPDATE_TO_HEAD)
+          version = yoctocommon.getVersionString("${params.RLS_VERSION}", "${params.IMAGE_TYPE}")
+          env.BUILD_VERSION = "${version}"
+          currentBuild.displayName = "${version}-${params.MACHINE}-${params.IMAGE_TYPE}" //replace Bitbake timestamp after building
+          printJobParameters()
+          yoctocommon.changeDistroVersion("${version}")
+          yoctocommon.syncSources("${env.BINARY_STORAGE_URL}", "${env.DOWNLOAD_DIR}")
+        }
+        writeFile file: 'VERSION', text: "${env.PACKAGE_NAME}: ${env.BUILD_VERSION}"
+      }
+    }
+
+    stage('clean') {
+      when { expression { return params.CLEAN_BUILD } }
+      steps {
+        dir ("${env.SHARED_BUILD}/tmp") { deleteDir() }
+        dir ("${env.SHARED_BUILD}/tmp-glibc") { deleteDir() }
+      }
+    }
+
+    stage('build') {
+      steps {
+        script {
+          build(params.IMAGE_TYPE, params.IMAGE_TYPE)
+          createArchive(params.IMAGE_TYPE, env.IMG_OUTPUT_DIR)
+          yoctocommon.archiveImages(env.IMG_OUTPUT_DIR)
+        }
+      }
+      post {
+        always {
+          script {
+            yoctocommon.syncSources("${env.DOWNLOAD_DIR}", "${env.BINARY_STORAGE_URL}")
+          }
+        }
+      }
+    }
+
+    stage('collect versions') {
+      steps {
+        script {
+          revisions = yoctocommon.getAutoRevHashes(params.IMAGE_TYPE)
+          writeFile(file: "${env.AUTOREV_VERSION_FILE}", text: "${revisions}")
+        }
+      }
+      post {
+        success {
+          archiveArtifacts(artifacts: "${env.SUBMODULE_VERION_FILE}, ${env.AUTOREV_VERSION_FILE}, ${env.DISTRO_VERSION_FILE}", onlyIfSuccessful: false)
+        }
+      }
+    }
+
+  } // stages
+}
+
+
+def printJobParameters() {
+  println "----------------------------------\n\
+  Job Parameters:\n\
+----------------------------------\n\
+    MACHINE        = ${params.MACHINE}\n\
+    IMAGE_TYPE     = ${params.IMAGE_TYPE}\n\
+    CLEAN_BUILD    = ${params.CLEAN_BUILD}\n\
+    UPDATE_TO_HEAD = ${params.DO_UPDATE_TO_HEAD}\n\
+    RLS_VERSION    = ${params.RLS_VERSION}\n\
+      --> version  = ${env.BUILD_VERSION}\n\
+----------------------------------\n"
+}
+
+
+def build(envType, imgType) {
+  def envPostFix = "${envType}" == "" ? "" : "-${envType}"
+  sh "bash -c '. ./env.image${envPostFix} && bitbake -k virtual/netmodule-image'"
+}
+
+
+def createArchive(imgType, outputDir) {
+  def imgTypePostfix = "${imgType}" == "" ? "" : "-${imgType}"
+  dir (outputDir) {
+    def image_basename = "netmodule-linux-image${imgTypePostfix}-${params.MACHINE}"
+    def basename_built = "${env.YOCTO_DEPLOYS}/${image_basename}"
+    def basename_archive = "./image${imgTypePostfix}-${params.MACHINE}"
+
+    sh "cp ${basename_built}.manifest ${basename_archive}.manifest"
+    sh "bash -c '${WORKSPACE}/openembedded-core/scripts/buildhistory-collect-srcrevs -p ${env.BUILD_HISTORY_DIR} > srcrev-${params.MACHINE}${imgTypePostfix}.inc'"
+
+    sh label: 'Copy License Manifest', returnStatus: true, script: """
+        LATEST_LICENSE_DIR=\$(ls -Artd ${env.BUILD_LICENSE_DIR}/netmodule-linux-image${imgTypePostfix}* | tail -n 1)
+        cp \$LATEST_LICENSE_DIR/license.manifest ${basename_archive}_license.manifest
+        """
+
+    if(imgType == "minimal") {
+      sh "cp ${env.YOCTO_DEPLOYS}/initramfs-linux/fitImage-${image_basename}-${params.MACHINE} fitImage-${image_basename}"
+    }
+    else {
+      sh "cp ${env.YOCTO_DEPLOYS}/fct-linux/fitImage-${image_basename}-${params.MACHINE} fitImage-${image_basename}"
+    }
+
+    sh "cp ${basename_built}.tar.gz ${basename_archive}.tar.gz"
+  }
+}

Jenkinsfile_mmcImages

@@ -0,0 +1,169 @@
+// Loading code requires a NODE context
+// But we want the code accessible outside the node Context
+// So declare yoctocommon (object created by the LOAD operation) outside the Node block.
+def yoctocommon
+
+
+// declarative pipeline
+pipeline {
+  agent {
+    node {
+      label 'lxbuild4'
+    }
+  }
+
+  parameters {
+    choice(name: 'MACHINE', choices: ['select...', 'am335x-nrhw20', 'am335x-nmhw21', 'imx8-nmhw23', 'am335x-nmhw24', 'am335x-hw25', 'am335x-hw26'], description: 'choose target platform')
+    choice(name: 'IMAGE_TYPE', choices: ['dev', 'bootloader', 'release'], description: 'choose image type')
+    string(name: 'RLS_VERSION', defaultValue: '', description: 'Set the version to build and use committed submodules')
+    booleanParam(name: 'CLEAN_BUILD', defaultValue: false, description: 'clean all temp directories before build starts')
+    booleanParam(name: 'DO_UPDATE_TO_HEAD', defaultValue: false, description: 'update submodules to head')
+  }
+
+  environment {
+    IMG_OUTPUT_DIR = "tmp/build-output"
+  }
+
+  options {
+    timeout(time: 8, unit: 'HOURS')
+    buildDiscarder(
+      logRotator(numToKeepStr: '50',
+                 daysToKeepStr: '3',
+                 artifactNumToKeepStr: '50',
+                 artifactDaysToKeepStr: '3'
+      )
+    )
+    disableConcurrentBuilds()
+  }
+
+  stages {
+    stage('prepare') {
+      steps {
+        script {
+          if("${params.MACHINE}" == "select...") {
+            error("Missing machine type --> select parameter MACHINE for a proper build")
+          }
+
+          // load yocto common file
+          env.ROOTDIR = pwd()
+          yoctocommon = load "${env.ROOTDIR}/Jenkinsfile_Common"
+
+          // Prepare Build Environment
+          env.YOCTO_DEPLOYS = "${env.BUILD_DEPLOY_DIR}/${params.MACHINE}"
+          yoctocommon.handleSubmodules(params.DO_UPDATE_TO_HEAD)
+          version = yoctocommon.getVersionString("${params.RLS_VERSION}", "${params.IMAGE_TYPE}")
+          env.BUILD_VERSION = "${version}"
+          currentBuild.displayName = "${version}-${params.MACHINE}-${params.IMAGE_TYPE}" //replace Bitbake timestamp after building
+          printJobParameters()
+          yoctocommon.changeDistroVersion("${version}")
+          yoctocommon.syncSources("${env.BINARY_STORAGE_URL}", "${env.DOWNLOAD_DIR}")
+        }
+        writeFile file: 'VERSION', text: "${env.PACKAGE_NAME}: ${env.BUILD_VERSION}"
+      }
+    }
+
+    stage('clean') {
+      when { expression { return params.CLEAN_BUILD } }
+      steps {
+        dir ("${SHARED_BUILD}/tmp") { deleteDir() }
+        dir ("${SHARED_BUILD}/tmp-glibc") { deleteDir() }
+      }
+    }
+
+    stage('build') {
+      steps {
+        script {
+          build(params.IMAGE_TYPE)
+          createArchive(params.IMAGE_TYPE, env.IMG_OUTPUT_DIR)
+          archiveOSTreeArtifact(env.IMG_OUTPUT_DIR)
+          yoctocommon.archiveImages(env.IMG_OUTPUT_DIR)
+        }
+      }
+      post {
+        always {
+          script {
+            yoctocommon.syncSources("${env.DOWNLOAD_DIR}", "${env.BINARY_STORAGE_URL}")
+          }
+        }
+      }
+    }
+
+    stage('collect versions') {
+      steps {
+        script {
+          revisions = yoctocommon.getAutoRevHashes('ostree')
+          writeFile(file: "${env.AUTOREV_VERSION_FILE}", text: "${revisions}")
+        }
+      }
+      post {
+        success {
+          archiveArtifacts(artifacts: "${env.SUBMODULE_VERION_FILE}, ${env.AUTOREV_VERSION_FILE}, ${env.DISTRO_VERSION_FILE}", onlyIfSuccessful: false)
+        }
+      }
+    }
+
+  } // stages
+}
+
+
+def printJobParameters() {
+  println "----------------------------------\n\
+  Job Parameters:\n\
+----------------------------------\n\
+    MACHINE        = ${params.MACHINE}\n\
+    IMAGE_TYPE     = ${params.IMAGE_TYPE}\n\
+    CLEAN_BUILD    = ${params.CLEAN_BUILD}\n\
+    UPDATE_TO_HEAD = ${params.DO_UPDATE_TO_HEAD}\n\
+    RLS_VERSION    = ${params.RLS_VERSION}\n\
+      --> version  = ${env.BUILD_VERSION}\n\
+----------------------------------\n"
+}
+
+def build(imgType) {
+  if(imgType == 'bootloader') {
+    sh "bash -c '. ./env.common && bitbake virtual/bootloader'"
+    return
+  }
+  def imgTypePostfix = "${imgType}" == "" ? "" : "-${imgType}"
+  sh "bash -c '. ./env.image-ostree && bitbake -k netmodule-linux-image${imgTypePostfix}'"
+}
+
+def createArchive(imgType, outputDir) {
+  dir (outputDir) {
+    if(imgType == 'bootloader') {
+      sh "cp ${env.YOCTO_DEPLOYS}/*u-boot-${params.MACHINE}*.img . || true"
+      sh "cp ${env.YOCTO_DEPLOYS}/*u-boot-${params.MACHINE}*.xmodem.bin . || true"
+      sh "cp ${env.YOCTO_DEPLOYS}/imx-boot . || true"
+      sh "cp ${env.YOCTO_DEPLOYS}/imx-boot.sd . || true"
+      return
+    }
+
+    def imgTypePostfix = "${imgType}" == "" ? "" : "-${imgType}"
+    def image_basename = "netmodule-linux-image${imgTypePostfix}-${params.MACHINE}"
+    def basename_built = "${env.YOCTO_DEPLOYS}/${image_basename}"
+    def basename_archive = "./image${imgTypePostfix}-${params.MACHINE}"
+
+    sh "cp ${basename_built}.manifest ${basename_archive}.manifest"
+    sh "bash -c '${WORKSPACE}/openembedded-core/scripts/buildhistory-collect-srcrevs -p ${env.BUILD_HISTORY_DIR} > srcrev-${params.MACHINE}${imgTypePostfix}.inc'"
+
+    sh label: 'Copy License Manifest', returnStatus: true, script: """
+        LATEST_LICENSE_DIR=\$(ls -Artd ${env.BUILD_LICENSE_DIR}/netmodule-linux-image${imgTypePostfix}* | tail -n 1)
+        cp \$LATEST_LICENSE_DIR/license.manifest ${basename_archive}_license.manifest
+        """
+
+    sh label: 'Copy initramfs License Manifest', returnStatus: true, script: """
+        LATEST_LICENSE_DIR=\$(ls -Artd ${env.BUILD_LICENSE_DIR}/initramfs-ostree-image-${params.MACHINE}-* | tail -n 1)
+        cp \$LATEST_LICENSE_DIR/license.manifest initramfs-ostree-image_license.manifest
+        """
+
+    sh "cp ${env.YOCTO_DEPLOYS}/fitImage-${params.MACHINE}.bin ."
+    sh "cp ${basename_built}.ota-ext4 ${basename_archive}.ota-ext4"
+    sh "cp ${basename_built}.wic ${basename_archive}.wic"
+    sh "tar czf ./ostree_repo${imgTypePostfix}.tar.gz -C ${env.YOCTO_DEPLOYS}/ostree_repo ."
+  }
+}
+
+def archiveOSTreeArtifact(outputDir) {
+  archiveArtifacts artifacts: "${outputDir}/ostree_repo*.tar.gz", onlyIfSuccessful: true
+  sh "rm -f ./${outputDir}/ostree_repo*.tar.gz"
+}