diff --git a/Jenkinsfile_Common b/Jenkinsfile_Common new file mode 100644 index 0000000..07f48eb --- /dev/null +++ b/Jenkinsfile_Common @@ -0,0 +1,158 @@ +// common parts for yocto builds + +env.SHARED_BUILD = "${WORKSPACE}/build" +env.BUILD_DEPLOY_DIR = "${env.SHARED_BUILD}/tmp/deploy/images" +env.BUILD_LICENSE_DIR = "${env.SHARED_BUILD}/tmp/deploy/licenses" +env.BUILD_HISTORY_DIR = "${env.SHARED_BUILD}/buildhistory" +env.DISTRO_VERSION_PATHNAME = "${env.SHARED_BUILD}/conf/distro_version.inc" +env.DOWNLOAD_DIR = "${env.SHARED_BUILD}/downloads" +env.BINARY_STORAGE_URL = "http://nmrepo.netmodule.intranet/src/yocto-downloads" +env.SUBMODULE_VERION_FILE = "submodule_revisions" +env.DISTRO_VERSION_FILE = "distro_version.inc" +env.AUTOREV_VERSION_FILE = "autorev_revisions.inc" +env.PACKAGE_NAME = "nm-os" + + +echo "loading common yocto build module..." + + +// Methods declared in external code are accessible +// directly from other code in the external file +// indirectly via the object created by the load operation +// eg. extcode.build(...) + + +def isRelease(versionParam) { + if((versionParam == "") || (versionParam == "latest")) { + return false + } + return true +} + + +def handleSubmodules(isForcedToHead) { + println "init submodules..." + sh 'git submodule init' + if(isForcedToHead) { + println "setting submodule hashes to head..." + sh 'git submodule update --remote --rebase' // update all submodules to HEAD + } + else { + sh 'git submodule update' // set all submodules to freezed commit + } + submoduleStatus = sh(returnStdout: true, script: "git submodule status").trim() // print submodule hashes to jenkins log + println "${submoduleStatus}" + writeFile(file: "${env.SUBMODULE_VERION_FILE}", text: "${submoduleStatus}") +} + + +def getTopUpstreamBuildNumber() { + // Iterating though all upstream jobs: + // currentBuild.upstreamBuilds.each { item -> + // echo "upstream build: ${item}" + // def nbr = item.getNumber() + // echo "nbr=${nbr}" + // } + def upstreamJobList = currentBuild.upstreamBuilds + def nbrOfUpstreamJobs = upstreamJobList.size() + if (nbrOfUpstreamJobs == 0) + return 0; + def topJob = upstreamJobList[nbrOfUpstreamJobs-1] + println "Top upstream project: " + topJob.getFullDisplayName() + def topJobNbr = topJob.getNumber() + println "Top upstream job build Number = ${topJobNbr}" + return topJobNbr +} + +def buildVersionString(imageType, actualBaseVersionString, versionParameter) { + // official release version + if(isRelease(versionParameter)) { + String newVersionStr = versionParameter + return newVersionStr + } + + // FCT release + if(imageType == 'fct') { + return actualBaseVersionString + } + + // nightly/incremental release + def buildnbr = getTopUpstreamBuildNumber() + String nightlyPart = actualBaseVersionString + ".Test${buildnbr}" + return nightlyPart +} + +def getVersionString(versionParam, imageType) { + sh 'git fetch -ap' + sh 'git fetch -t' + def gitCmd = "git describe --tags" + + if(!isRelease(versionParam)) { + gitCmd = "${gitCmd} --dirty" + } + gitversion = sh(returnStdout: true, script: "${gitCmd}").trim() + String[] versionArr = "${gitversion}".split("-") + + versionArr[0] = buildVersionString(imageType, versionArr[0], versionParam) + rlsVersion = versionArr.join("-") + return rlsVersion +} + + +def changeDistroVersion(versionString){ + println "Set the distro version to ${versionString}..." + def versionTag = "DISTRO_VERSION = \"${versionString}\"" + writeFile(file: "${env.DISTRO_VERSION_PATHNAME}", text: "${versionTag}") + sh(script: "cp ${env.DISTRO_VERSION_PATHNAME} ${env.DISTRO_VERSION_FILE}") +} + +def cleanupDistroVersion() { + println "cleaning repository regarding distro version..." + sh(script:"git clean -f ${env.DISTRO_VERSION_PATHNAME}") +} + + +def archiveImages(imgageDir) { + dir ('tmp/artifacts') { + zip archive: true, dir: "${WORKSPACE}/${imgageDir}", glob: "*", zipFile: "${env.PACKAGE_NAME}-${env.BUILD_VERSION}-${params.MACHINE}-${params.IMAGE_TYPE}.zip" + } + sh "rm -rf ${WORKSPACE}/tmp" +} + + +def syncSources(src, dst) { + def hasSrcUrl = (src.contains("http")) + def from = src + def to = dst + + // convert the URL into ssh syntax: + def url = (hasSrcUrl) ? src : dst + String[] repoParts = url.split("//")[1].split("/") + repoParts[0] = "build_user@" + repoParts[0] + ":/repo/repo" + sshSrc = repoParts.join("/") + + if(hasSrcUrl) { + println "getting data from server..." + from = sshSrc + } + else { + println "putting data to server..." + to = sshSrc + } + + sshagent (credentials: ['7767e711-08a4-4c71-b080-197253dd7392']) { + sh "rsync -auvz --ignore-existing -e \"ssh\" ${from}/* ${to}" + } +} + + +def getAutoRevHashes(envType) { + def env = "${envType}" == "" ? "" : "-${envType}" + def revs = sh(returnStdout: true, script: "bash -c \". ./env.image${env} > /dev/null && buildhistory-collect-srcrevs\"").trim() + return revs +} + + +// !!Important Boilerplate!! +// The external code must return it's contents as an object +return this; diff --git a/Jenkinsfile_RamdiskImages b/Jenkinsfile_RamdiskImages new file mode 100644 index 0000000..f672d8e --- /dev/null +++ b/Jenkinsfile_RamdiskImages @@ -0,0 +1,152 @@ +// Loading code requires a NODE context +// But we want the code accessible outside the node Context +// So declare yoctocommon (object created by the LOAD operation) outside the Node block. +def yoctocommon + + +// declarative pipeline +pipeline { + agent { + node { + label 'lxbuild3' + } + } + + parameters { + choice(name: 'MACHINE', choices: ['select...', 'am335x-nrhw20', 'am335x-nmhw21', 'imx8-nmhw23', 'am335x-nmhw24', 'am335x-hw25', 'am335x-hw26'], description: 'choose target platform') + choice(name: 'IMAGE_TYPE', choices: ['lava', 'fct', 'minimal'], description: 'choose image type') + string(name: 'RLS_VERSION', defaultValue: '', description: 'Set the version to build and use committed submodules') + booleanParam(name: 'CLEAN_BUILD', defaultValue: false, description: 'clean all temp directories before build starts') + booleanParam(name: 'DO_UPDATE_TO_HEAD', defaultValue: false, description: 'update submodules to head') + } + + environment { + IMG_OUTPUT_DIR = "tmp/build-output" + } + + options { + timeout(time: 8, unit: 'HOURS') + buildDiscarder( + logRotator(numToKeepStr: '50', + daysToKeepStr: '3', + artifactNumToKeepStr: '50', + artifactDaysToKeepStr: '3' + ) + ) + disableConcurrentBuilds() + } + + stages { + stage('prepare') { + steps { + script { + if("${params.MACHINE}" == "select...") { + error("Missing machine type --> select parameter MACHINE for a proper build") + } + + // load yocto common file + env.ROOTDIR = pwd() + yoctocommon = load "${env.ROOTDIR}/Jenkinsfile_Common" + + // Prepare Build Environment + env.YOCTO_DEPLOYS = "${env.BUILD_DEPLOY_DIR}/${params.MACHINE}" + yoctocommon.handleSubmodules(params.DO_UPDATE_TO_HEAD) + version = yoctocommon.getVersionString("${params.RLS_VERSION}", "${params.IMAGE_TYPE}") + env.BUILD_VERSION = "${version}" + currentBuild.displayName = "${version}-${params.MACHINE}-${params.IMAGE_TYPE}" //replace Bitbake timestamp after building + printJobParameters() + yoctocommon.changeDistroVersion("${version}") + yoctocommon.syncSources("${env.BINARY_STORAGE_URL}", "${env.DOWNLOAD_DIR}") + } + writeFile file: 'VERSION', text: "${env.PACKAGE_NAME}: ${env.BUILD_VERSION}" + } + } + + stage('clean') { + when { expression { return params.CLEAN_BUILD } } + steps { + dir ("${env.SHARED_BUILD}/tmp") { deleteDir() } + dir ("${env.SHARED_BUILD}/tmp-glibc") { deleteDir() } + } + } + + stage('build') { + steps { + script { + build(params.IMAGE_TYPE, params.IMAGE_TYPE) + createArchive(params.IMAGE_TYPE, env.IMG_OUTPUT_DIR) + yoctocommon.archiveImages(env.IMG_OUTPUT_DIR) + } + } + post { + always { + script { + yoctocommon.syncSources("${env.DOWNLOAD_DIR}", "${env.BINARY_STORAGE_URL}") + } + } + } + } + + stage('collect versions') { + steps { + script { + revisions = yoctocommon.getAutoRevHashes(params.IMAGE_TYPE) + writeFile(file: "${env.AUTOREV_VERSION_FILE}", text: "${revisions}") + } + } + post { + success { + archiveArtifacts(artifacts: "${env.SUBMODULE_VERION_FILE}, ${env.AUTOREV_VERSION_FILE}, ${env.DISTRO_VERSION_FILE}", onlyIfSuccessful: false) + } + } + } + + } // stages +} + + +def printJobParameters() { + println "----------------------------------\n\ + Job Parameters:\n\ +----------------------------------\n\ + MACHINE = ${params.MACHINE}\n\ + IMAGE_TYPE = ${params.IMAGE_TYPE}\n\ + CLEAN_BUILD = ${params.CLEAN_BUILD}\n\ + UPDATE_TO_HEAD = ${params.DO_UPDATE_TO_HEAD}\n\ + RLS_VERSION = ${params.RLS_VERSION}\n\ + --> version = ${env.BUILD_VERSION}\n\ +----------------------------------\n" +} + + +def build(envType, imgType) { + def envPostFix = "${envType}" == "" ? "" : "-${envType}" + sh "bash -c '. ./env.image${envPostFix} && bitbake -k virtual/netmodule-image'" +} + + +def createArchive(imgType, outputDir) { + def imgTypePostfix = "${imgType}" == "" ? "" : "-${imgType}" + dir (outputDir) { + def image_basename = "netmodule-linux-image${imgTypePostfix}-${params.MACHINE}" + def basename_built = "${env.YOCTO_DEPLOYS}/${image_basename}" + def basename_archive = "./image${imgTypePostfix}-${params.MACHINE}" + + sh "cp ${basename_built}.manifest ${basename_archive}.manifest" + sh "bash -c '${WORKSPACE}/openembedded-core/scripts/buildhistory-collect-srcrevs -p ${env.BUILD_HISTORY_DIR} > srcrev-${params.MACHINE}${imgTypePostfix}.inc'" + + sh label: 'Copy License Manifest', returnStatus: true, script: """ + LATEST_LICENSE_DIR=\$(ls -Artd ${env.BUILD_LICENSE_DIR}/netmodule-linux-image${imgTypePostfix}* | tail -n 1) + cp \$LATEST_LICENSE_DIR/license.manifest ${basename_archive}_license.manifest + """ + + if(imgType == "minimal") { + sh "cp ${env.YOCTO_DEPLOYS}/initramfs-linux/fitImage-${image_basename}-${params.MACHINE} fitImage-${image_basename}" + } + else { + sh "cp ${env.YOCTO_DEPLOYS}/fct-linux/fitImage-${image_basename}-${params.MACHINE} fitImage-${image_basename}" + } + + sh "cp ${basename_built}.tar.gz ${basename_archive}.tar.gz" + } +} diff --git a/Jenkinsfile_mmcImages b/Jenkinsfile_mmcImages new file mode 100644 index 0000000..55049e4 --- /dev/null +++ b/Jenkinsfile_mmcImages @@ -0,0 +1,169 @@ +// Loading code requires a NODE context +// But we want the code accessible outside the node Context +// So declare yoctocommon (object created by the LOAD operation) outside the Node block. +def yoctocommon + + +// declarative pipeline +pipeline { + agent { + node { + label 'lxbuild4' + } + } + + parameters { + choice(name: 'MACHINE', choices: ['select...', 'am335x-nrhw20', 'am335x-nmhw21', 'imx8-nmhw23', 'am335x-nmhw24', 'am335x-hw25', 'am335x-hw26'], description: 'choose target platform') + choice(name: 'IMAGE_TYPE', choices: ['dev', 'bootloader', 'release'], description: 'choose image type') + string(name: 'RLS_VERSION', defaultValue: '', description: 'Set the version to build and use committed submodules') + booleanParam(name: 'CLEAN_BUILD', defaultValue: false, description: 'clean all temp directories before build starts') + booleanParam(name: 'DO_UPDATE_TO_HEAD', defaultValue: false, description: 'update submodules to head') + } + + environment { + IMG_OUTPUT_DIR = "tmp/build-output" + } + + options { + timeout(time: 8, unit: 'HOURS') + buildDiscarder( + logRotator(numToKeepStr: '50', + daysToKeepStr: '3', + artifactNumToKeepStr: '50', + artifactDaysToKeepStr: '3' + ) + ) + disableConcurrentBuilds() + } + + stages { + stage('prepare') { + steps { + script { + if("${params.MACHINE}" == "select...") { + error("Missing machine type --> select parameter MACHINE for a proper build") + } + + // load yocto common file + env.ROOTDIR = pwd() + yoctocommon = load "${env.ROOTDIR}/Jenkinsfile_Common" + + // Prepare Build Environment + env.YOCTO_DEPLOYS = "${env.BUILD_DEPLOY_DIR}/${params.MACHINE}" + yoctocommon.handleSubmodules(params.DO_UPDATE_TO_HEAD) + version = yoctocommon.getVersionString("${params.RLS_VERSION}", "${params.IMAGE_TYPE}") + env.BUILD_VERSION = "${version}" + currentBuild.displayName = "${version}-${params.MACHINE}-${params.IMAGE_TYPE}" //replace Bitbake timestamp after building + printJobParameters() + yoctocommon.changeDistroVersion("${version}") + yoctocommon.syncSources("${env.BINARY_STORAGE_URL}", "${env.DOWNLOAD_DIR}") + } + writeFile file: 'VERSION', text: "${env.PACKAGE_NAME}: ${env.BUILD_VERSION}" + } + } + + stage('clean') { + when { expression { return params.CLEAN_BUILD } } + steps { + dir ("${SHARED_BUILD}/tmp") { deleteDir() } + dir ("${SHARED_BUILD}/tmp-glibc") { deleteDir() } + } + } + + stage('build') { + steps { + script { + build(params.IMAGE_TYPE) + createArchive(params.IMAGE_TYPE, env.IMG_OUTPUT_DIR) + archiveOSTreeArtifact(env.IMG_OUTPUT_DIR) + yoctocommon.archiveImages(env.IMG_OUTPUT_DIR) + } + } + post { + always { + script { + yoctocommon.syncSources("${env.DOWNLOAD_DIR}", "${env.BINARY_STORAGE_URL}") + } + } + } + } + + stage('collect versions') { + steps { + script { + revisions = yoctocommon.getAutoRevHashes('ostree') + writeFile(file: "${env.AUTOREV_VERSION_FILE}", text: "${revisions}") + } + } + post { + success { + archiveArtifacts(artifacts: "${env.SUBMODULE_VERION_FILE}, ${env.AUTOREV_VERSION_FILE}, ${env.DISTRO_VERSION_FILE}", onlyIfSuccessful: false) + } + } + } + + } // stages +} + + +def printJobParameters() { + println "----------------------------------\n\ + Job Parameters:\n\ +----------------------------------\n\ + MACHINE = ${params.MACHINE}\n\ + IMAGE_TYPE = ${params.IMAGE_TYPE}\n\ + CLEAN_BUILD = ${params.CLEAN_BUILD}\n\ + UPDATE_TO_HEAD = ${params.DO_UPDATE_TO_HEAD}\n\ + RLS_VERSION = ${params.RLS_VERSION}\n\ + --> version = ${env.BUILD_VERSION}\n\ +----------------------------------\n" +} + +def build(imgType) { + if(imgType == 'bootloader') { + sh "bash -c '. ./env.common && bitbake virtual/bootloader'" + return + } + def imgTypePostfix = "${imgType}" == "" ? "" : "-${imgType}" + sh "bash -c '. ./env.image-ostree && bitbake -k netmodule-linux-image${imgTypePostfix}'" +} + +def createArchive(imgType, outputDir) { + dir (outputDir) { + if(imgType == 'bootloader') { + sh "cp ${env.YOCTO_DEPLOYS}/*u-boot-${params.MACHINE}*.img . || true" + sh "cp ${env.YOCTO_DEPLOYS}/*u-boot-${params.MACHINE}*.xmodem.bin . || true" + sh "cp ${env.YOCTO_DEPLOYS}/imx-boot . || true" + sh "cp ${env.YOCTO_DEPLOYS}/imx-boot.sd . || true" + return + } + + def imgTypePostfix = "${imgType}" == "" ? "" : "-${imgType}" + def image_basename = "netmodule-linux-image${imgTypePostfix}-${params.MACHINE}" + def basename_built = "${env.YOCTO_DEPLOYS}/${image_basename}" + def basename_archive = "./image${imgTypePostfix}-${params.MACHINE}" + + sh "cp ${basename_built}.manifest ${basename_archive}.manifest" + sh "bash -c '${WORKSPACE}/openembedded-core/scripts/buildhistory-collect-srcrevs -p ${env.BUILD_HISTORY_DIR} > srcrev-${params.MACHINE}${imgTypePostfix}.inc'" + + sh label: 'Copy License Manifest', returnStatus: true, script: """ + LATEST_LICENSE_DIR=\$(ls -Artd ${env.BUILD_LICENSE_DIR}/netmodule-linux-image${imgTypePostfix}* | tail -n 1) + cp \$LATEST_LICENSE_DIR/license.manifest ${basename_archive}_license.manifest + """ + + sh label: 'Copy initramfs License Manifest', returnStatus: true, script: """ + LATEST_LICENSE_DIR=\$(ls -Artd ${env.BUILD_LICENSE_DIR}/initramfs-ostree-image-${params.MACHINE}-* | tail -n 1) + cp \$LATEST_LICENSE_DIR/license.manifest initramfs-ostree-image_license.manifest + """ + + sh "cp ${env.YOCTO_DEPLOYS}/fitImage-${params.MACHINE}.bin ." + sh "cp ${basename_built}.ota-ext4 ${basename_archive}.ota-ext4" + sh "cp ${basename_built}.wic ${basename_archive}.wic" + sh "tar czf ./ostree_repo${imgTypePostfix}.tar.gz -C ${env.YOCTO_DEPLOYS}/ostree_repo ." + } +} + +def archiveOSTreeArtifact(outputDir) { + archiveArtifacts artifacts: "${outputDir}/ostree_repo*.tar.gz", onlyIfSuccessful: true + sh "rm -f ./${outputDir}/ostree_repo*.tar.gz" +}