diff --git a/app-ios/fastlane/Fastfile b/app-ios/fastlane/Fastfile index a873aea71fce..b9a961b2e977 100644 --- a/app-ios/fastlane/Fastfile +++ b/app-ios/fastlane/Fastfile @@ -16,8 +16,8 @@ default_platform(:ios) platform :ios do - desc "Push a new prod release to AppStore" - lane :appstore_prod do |options| + desc "Build a new Mail prod release with AppStore configuration" + lane :build_mail_prod do |options| match( app_identifier: ["de.tutao.tutanota", "de.tutao.tutanota.TutanotaShareExtension", "de.tutao.tutanota.TutanotaNotificationExtension"], type: "appstore", @@ -37,18 +37,23 @@ platform :ios do include_symbols: true, verbose: true ) - if options[:submit] - upload_to_app_store( - skip_screenshots: true, - submit_for_review: false, - precheck_include_in_app_purchases: false, - # must use force as long as we don't automatically create html previews - force: true, - api_key_path: ENV["API_KEY_JSON_FILE_PATH"] - ) - end end + desc "Publish a Mail artifact to AppStore" + lane :publish_mail_prod do |options| + sh 'echo "Uploading mail artifact ' + options[:file] + '"' + + upload_to_app_store( + ipa: options[:file], + skip_screenshots: true, + submit_for_review: false, + precheck_include_in_app_purchases: false, + # must use force as long as we don't automatically create html previews + force: true, + api_key_path: ENV["API_KEY_JSON_FILE_PATH"] + ) + end + desc "Build a new prod release for ad-hoc" lane :adhoc_prod do |options| match( @@ -77,7 +82,7 @@ platform :ios do desc "Push a new staging release to TestFlight" - lane :testflight_staging do + lane :build_mail_staging do match( app_identifier: ["de.tutao.tutanota.test", "de.tutao.tutanota.test.TutanotaShareExtension", "de.tutao.tutanota.test.TutanotaNotificationExtension"], type: "appstore", @@ -96,13 +101,19 @@ platform :ios do output_name: "tutanota-" + get_version_number(target: "tutanota") + "-test", verbose: true ) + end + + desc "Publish Mail staging to TestFlight" + lane :publish_mail_staging do |options| + sh 'echo "Uploading mail staging to TestFlight ' + options[:file] + '"' upload_to_testflight( + ipa: options[:file], app_identifier: "de.tutao.tutanota.test", skip_submission: true, api_key_path: ENV["API_KEY_JSON_FILE_PATH"] ) - end + end desc "Build a new staging release for ad-hoc" lane :adhoc_staging do diff --git a/buildSrc/createReleaseDraft.js b/buildSrc/createReleaseDraft.js index a51853585890..0792326bc022 100644 --- a/buildSrc/createReleaseDraft.js +++ b/buildSrc/createReleaseDraft.js @@ -29,17 +29,6 @@ if (wasRunFromCli) { } async function run({ name, tag, notes, uploadFile, dryRun, toFile }) { - const releaseToken = process.env.GITHUB_TOKEN - - if (!releaseToken) { - throw new Error("No GITHUB_TOKEN set!") - } - - const octokit = new Octokit({ - auth: releaseToken, - userAgent: "tuta-github-release-v0.0.1", - }) - notes = renderCompleteNotes({ notes: await fs.promises.readFile(notes, { encoding: "utf8" }), files: uploadFile }) if (toFile) { @@ -48,6 +37,17 @@ async function run({ name, tag, notes, uploadFile, dryRun, toFile }) { } else if (dryRun) { console.log(`dry run, so not creating draft with release notes\n\n${notes}\nand name ${name}, tag ${tag} \n ${uploadFile}`) } else { + const releaseToken = process.env.GITHUB_TOKEN + + if (!releaseToken) { + throw new Error("No GITHUB_TOKEN set!") + } + + const octokit = new Octokit({ + auth: releaseToken, + userAgent: "tuta-github-release-v0.0.1", + }) + const draftResponse = await createReleaseDraft(octokit, name, tag, notes) const { upload_url, id } = draftResponse.data diff --git a/ci/Android.Jenkinsfile b/ci/Android.Jenkinsfile index 49cf94df7125..8b840a1329c0 100644 --- a/ci/Android.Jenkinsfile +++ b/ci/Android.Jenkinsfile @@ -5,7 +5,8 @@ pipeline { PATH = "${env.NODE_PATH}:${env.PATH}:/home/jenkins/emsdk/upstream/bin/:/home/jenkins/emsdk/:/home/jenkins/emsdk/upstream/emscripten" ANDROID_SDK_ROOT = "/opt/android-sdk-linux" ANDROID_HOME = "/opt/android-sdk-linux" - GITHUB_RELEASE_PAGE = "https://github.com/tutao/tutanota/releases/tag/tutanota-android-release-${VERSION}" + STAGING_FILE_PATH = "build/app-android/tutanota-app-tutao-releaseTest-${VERSION}.apk" + PROD_FILE_PATH = "build/app-android/tutanota-app-tutao-release-${VERSION}.apk" } agent { @@ -18,19 +19,32 @@ pipeline { parameters { booleanParam( - name: 'RELEASE', defaultValue: false, - description: "Build a test and release version of the app. " + - "Uploads both to Nexus and creates a new release on google play, " + - "which must be manually published from play.google.com/console" + name: 'UPLOAD', + defaultValue: false, + description: "Upload staging/prod to Nexus" + ) + booleanParam( + name: 'STAGING', + defaultValue: true + ) + booleanParam( + name: 'PROD', + defaultValue: true ) - persistentText( - name: "releaseNotes", - defaultValue: "", - description: "release notes for this build" - ) } stages { + stage("Checking params") { + steps { + script{ + if(!params.STAGING && !params.PROD) { + currentBuild.result = 'ABORTED' + error('No artifacts were selected.') + } + } + echo "Params OKAY" + } + } // stage checking params stage('Check Github') { steps { script { @@ -47,15 +61,14 @@ pipeline { } } stage('Build') { - stages { - stage('Testing') { + parallel { + stage('Staging') { + when { expression { return params.STAGING } } environment { APK_SIGN_ALIAS = "test.tutao.de" } - agent { - label 'linux' - } steps { + echo "Building STAGING ${VERSION}" sh 'npm ci' sh 'npm run build-packages' script { @@ -72,18 +85,16 @@ pipeline { ]) { sh 'node android.js -b releaseTest test' } - stash includes: "build/app-android/tutanota-app-tutao-releaseTest-${VERSION}.apk", name: 'apk-testing' - } - } // stage testing + stash includes: "${STAGING_FILE_PATH}", name: 'apk-staging' + } // steps + } // stage staging stage('Production') { - when { - expression { return params.RELEASE } - } + when { expression { return params.PROD } } environment { APK_SIGN_ALIAS = "tutao.de" } steps { - echo "Building ${VERSION}" + echo "Building PROD ${VERSION}" sh 'npm ci' sh 'npm run build-packages' script { @@ -100,112 +111,44 @@ pipeline { ]) { sh 'node android.js -b release prod' } - stash includes: "build/app-android/tutanota-app-tutao-release-${VERSION}.apk", name: 'apk-production' + stash includes: "${PROD_FILE_PATH}", name: 'apk-production' } } // stage production - } - } + } // stages + } // stage build - stage('Publish') { - when { - expression { return params.RELEASE } - } - stages { - stage('Testing') { + stage('Upload to Nexus') { + when { expression { return params.UPLOAD } } + parallel { + stage('Staging') { + when { expression { return params.STAGING } } steps { - script { - def util = load "ci/jenkins-lib/util.groovy" - unstash 'apk-testing' - - util.publishToNexus( - groupId: "app", - artifactId: "android-test", - version: "${VERSION}", - assetFilePath: "${WORKSPACE}/build/app-android/tutanota-app-tutao-releaseTest-${VERSION}.apk", - fileExtension: 'apk' - ) - - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'Failed to upload android test app to Play Store') { - // This doesn't publish to the main app on play store, - // instead it gets published to the hidden "tutanota-test" app - // this happens because the AppId is set to de.tutao.tutanota.test by the android build - // and play store knows which app to publish just based on the id - androidApkUpload( - googleCredentialsId: 'android-app-publisher-credentials', - apkFilesPattern: "build/app-android/tutanota-app-tutao-releaseTest-${VERSION}.apk", - trackName: 'internal', - rolloutPercentage: '100%', - recentChangeList: [ - [ - language: "en-US", - text : "see: ${GITHUB_RELEASE_PAGE}" - ] - ] - ) // androidApkUpload - } // catchError - - } + unstash 'apk-staging' + uploadToNexus("android-test", STAGING_FILE_PATH) } - } // stage testing + } // stage staging stage('Production') { + when { expression { return params.PROD } } steps { - sh 'npm ci' unstash 'apk-production' - - script { - def filePath = "build/app-android/tutanota-app-tutao-release-${VERSION}.apk" - def util = load "ci/jenkins-lib/util.groovy" - - util.publishToNexus( - groupId: "app", - artifactId: "android", - version: "${VERSION}", - assetFilePath: "${WORKSPACE}/${filePath}", - fileExtension: 'apk' - ) - - androidApkUpload( - googleCredentialsId: 'android-app-publisher-credentials', - apkFilesPattern: "${filePath}", - trackName: 'production', - // Don't publish the app to users directly - // It will require manual intervention at play.google.com/console - rolloutPercentage: '0%', - recentChangeList: [ - [ - language: "en-US", - text : "see: ${GITHUB_RELEASE_PAGE}" - ] - ] - ) - } + uploadToNexus("android", PROD_FILE_PATH) } } // stage production - } - } - stage('Tag and publish release page') { - when { - expression { return params.RELEASE } - } - steps { - // Needed to upload it - unstash 'apk-production' + } // stages + } // stage upload to nexus + } // stages +} // pipeline - script { - def filePath = "build/app-android/tutanota-app-tutao-release-${VERSION}.apk" +def uploadToNexus(String artifactId, String filePath) { + script { + def util = load "ci/jenkins-lib/util.groovy" - writeFile file: "notes.txt", text: params.releaseNotes - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'Failed to create github release page for android') { - withCredentials([string(credentialsId: 'github-access-token', variable: 'GITHUB_TOKEN')]) { - sh """node buildSrc/createReleaseDraft.js --name '${VERSION} (Android)' \ - --tag 'tutanota-android-release-${VERSION}' \ - --uploadFile '${WORKSPACE}/${filePath}' \ - --notes notes.txt""" - } // withCredentials - } // catchError - sh "rm notes.txt" - } // script - } - } + util.publishToNexus( + groupId: "app", + artifactId: "${artifactId}", + version: "${VERSION}", + assetFilePath: "${WORKSPACE}/${filePath}", + fileExtension: 'apk' + ) } -} +} \ No newline at end of file diff --git a/ci/Desktop.Jenkinsfile b/ci/Desktop.Jenkinsfile index 2908a83c4abe..bfa9dda6332d 100644 --- a/ci/Desktop.Jenkinsfile +++ b/ci/Desktop.Jenkinsfile @@ -10,15 +10,25 @@ pipeline { parameters { booleanParam( - name: 'RELEASE', + name: 'UPLOAD', defaultValue: false, - description: "Prepare a release version (doesn't publish to production, this is done manually)" + description: "Upload built clients to Nexus" + ) + booleanParam( + name: 'WINDOWS', + defaultValue: true, + description: "Build Windows client" + ) + booleanParam( + name: 'MAC', + defaultValue: true, + description: "Build Mac client" + ) + booleanParam( + name: 'LINUX', + defaultValue: true, + description: "Build Linux client" ) - persistentText( - name: "releaseNotes", - defaultValue: "", - description: "release notes for this build" - ) } agent { @@ -26,6 +36,17 @@ pipeline { } stages { + stage("Checking params") { + steps { + script{ + if(!params.WINDOWS && !params.MAC && !params.LINUX) { + currentBuild.result = 'ABORTED' + error('No artifacts were selected.') + } + } + echo "Params OKAY" + } + } // stage checking params stage('Check Github') { steps { script { @@ -33,74 +54,74 @@ pipeline { util.checkGithub() } } - } - stage('Build dependencies') { - parallel { - stage('Build webapp') { - agent { - dockerfile { - filename 'linux-build.dockerfile' - label 'master' - dir 'ci/containers' - additionalBuildArgs "--format docker" - args '--network host' - } // docker - } // agent - steps { - sh 'npm ci' - sh 'npm run build-packages' - sh 'node webapp.js release' - - // excluding web-specific and mobile specific parts which we don't need in desktop - stash includes: 'build/**', excludes: '**/braintree.html, **/index.html, **/app.html, **/desktop.html, **/index-index.js, **/index-app.js, **/index-desktop.js, **/sw.js', name: 'web_base' - } - } - - stage('Native modules') { - agent { - label 'win-native' - } - steps { - bat "npm ci" + } // stage check github + stage('Build webapp') { + agent { + dockerfile { + filename 'linux-build.dockerfile' + label 'master' + dir 'ci/containers' + additionalBuildArgs "--format docker" + args '--network host' + } // docker + } // agent + steps { + sh 'npm ci' + sh 'npm run build-packages' + sh 'node webapp.js release' - bat "node buildSrc\\getNativeLibrary.js better-sqlite3 --copy-target better_sqlite3 --force-rebuild --root-dir ${WORKSPACE}" - stash includes: 'native-cache/**/*', name: 'native_modules' - } - } + // excluding web-specific and mobile specific parts which we don't need in desktop + stash includes: 'build/**', excludes: '**/braintree.html, **/index.html, **/app.html, **/desktop.html, **/index-index.js, **/index-app.js, **/index-desktop.js, **/sw.js', name: 'web_base' } - } - + } // stage build webapp stage('Build desktop clients') { parallel { stage('Windows') { - environment { - PATH = "${env.NODE_PATH}:${env.PATH}" - } - agent { - label 'win-cross-compile' - } - steps { - initBuildArea() - - // nativeLibraryProvider.js placed the built native modules in the correct location (native-cache) - // so they will be picked up by our rollup plugin - unstash 'native_modules' + when { expression { return params.WINDOWS } } + stages { + stage('Native modules') { + agent { + label 'win-native' + } + steps { + bat "npm ci" - // add DEBUG for electron-builder because it tends to not let us know about things failing - withCredentials([string(credentialsId: 'YUBI_28989236_PIN', variable: 'PW')]) { - sh ''' - export YUBI_PIN=${PW}; - DEBUG=electron-builder node desktop --existing --platform win ''' + bat "node buildSrc\\getNativeLibrary.js better-sqlite3 --copy-target better_sqlite3 --force-rebuild --root-dir ${WORKSPACE}" + stash includes: 'native-cache/**/*', name: 'native_modules' + } } + stage("Client") { + environment { + PATH = "${env.NODE_PATH}:${env.PATH}" + } + agent { + label 'win-cross-compile' + } + steps { + initBuildArea() - dir('artifacts') { - stash includes: 'desktop-test/*', name:'win_installer_test' - stash includes: 'desktop/*', name:'win_installer' - } - } - } + // nativeLibraryProvider.js placed the built native modules in the correct location (native-cache) + // so they will be picked up by our rollup plugin + unstash 'native_modules' + + // add DEBUG for electron-builder because it tends to not let us know about things failing + withCredentials([string(credentialsId: 'YUBI_28989236_PIN', variable: 'PW')]) { + sh ''' + export YUBI_PIN=${PW}; + DEBUG=electron-builder node desktop --existing --platform win ''' + } + + dir('artifacts') { + stash includes: 'desktop-test/*', name:'win_installer_test' + stash includes: 'desktop/*', name:'win_installer' + } + } // steps + } // stage client + } // stages + } // stage windows stage('Mac') { + when { expression { return params.MAC } } environment { PATH = "${env.NODE_MAC_PATH}:${env.PATH}" } @@ -117,24 +138,25 @@ pipeline { ]) { sh 'security unlock-keychain -p $FASTLANE_KEYCHAIN_PASSWORD' script { - def stage = params.RELEASE ? 'release' : 'prod' + def stage = params.UPLOAD ? 'release' : 'prod' sh ''' export APPLEID=${APPLEIDVAR}; export APPLEIDPASS=${APPLEIDPASSVAR}; export APPLETEAMID=${APPLETEAMIDVAR}; node desktop --existing --architecture universal --platform mac ''' + "${stage}" dir('artifacts') { - if (params.RELEASE) { + if (params.UPLOAD) { stash includes: 'desktop-test/*', name:'mac_installer_test' } stash includes: 'desktop/*', name:'mac_installer' } } - } - } - } + } // withCredentials + } // steps + } // stage mac stage('Linux') { + when { expression { return params.LINUX } } agent { dockerfile { filename 'linux-build.dockerfile' @@ -153,13 +175,12 @@ pipeline { stash includes: 'desktop-test/*', name:'linux_installer_test' stash includes: 'desktop/*', name:'linux_installer' } - } - } - } - } + } // steps + } // stage linux + } // stages + } // stage build desktop clients - stage('Preparation for build deb and publish') { - when { expression { return params.RELEASE } } + stage('Preparation for sign clients and upload to Nexus') { agent { label 'master' } @@ -170,8 +191,8 @@ pipeline { } } } - stage('Build deb and publish') { - when { expression { return params.RELEASE } } + stage('Sign clients and upload to Nexus') { + when { expression { return params.UPLOAD } } agent { dockerfile { filename 'linux-build.dockerfile' @@ -181,99 +202,120 @@ pipeline { args "--network host -v /run:/run:rw,z -v /opt/repository:/opt/repository:rw,z --device=${env.DEVICE_PATH}" } // docker } - environment { PATH = "${env.NODE_PATH}:${env.PATH}" } - steps { - sh 'npm ci' - sh 'npm run build-packages' - sh 'rm -rf ./build/*' - - dir('build') { - unstash 'linux_installer' - unstash 'mac_installer' - unstash 'win_installer' - unstash 'linux_installer_test' - unstash 'mac_installer_test' - unstash 'win_installer_test' - } - - withCredentials([string(credentialsId: 'HSM_USER_PIN', variable: 'PW')]) { - sh '''export HSM_USER_PIN=${PW}; node buildSrc/signDesktopClients.js''' - } + environment { + PATH = "${env.NODE_PATH}:${env.PATH}" + } + stages { + stage('Preparation for sign and upload') { + steps { + sh 'npm ci' + sh 'npm run build-packages' + } + } + stage('Sign and upload') { + parallel { + stage('Windows') { + when { expression { return params.WINDOWS } } + steps { + dir('build') { + unstash 'win_installer' + unstash 'win_installer_test' + } - sh 'node buildSrc/publish.js desktop' + withCredentials([string(credentialsId: 'HSM_USER_PIN', variable: 'PW')]) { + sh '''export HSM_USER_PIN=${PW}; node buildSrc/signDesktopClients.js''' + } - script { // create release draft - def desktopLinux = "build/desktop/tutanota-desktop-linux.AppImage" - def desktopWin = "build/desktop/tutanota-desktop-win.exe" - def desktopMac = "build/desktop/tutanota-desktop-mac.dmg" + uploadWindowsArtifacts() + } + } // stage windows + stage('Mac') { + when { expression { return params.MAC } } + steps { + dir('build') { + unstash 'mac_installer' + unstash 'mac_installer_test' + } - writeFile file: "notes.txt", text: params.releaseNotes - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'Failed to create github release page for desktop') { - withCredentials([string(credentialsId: 'github-access-token', variable: 'GITHUB_TOKEN')]) { - sh """node buildSrc/createReleaseDraft.js --name '${VERSION} (Desktop)' \ - --tag 'tutanota-desktop-release-${VERSION}' \ - --uploadFile '${WORKSPACE}/${desktopLinux}' \ - --uploadFile '${WORKSPACE}/${desktopWin}' \ - --uploadFile '${WORKSPACE}/${desktopMac}' \ - --notes notes.txt""" - } // withCredentials - } // catchError - sh "rm notes.txt" - } // script release draft + withCredentials([string(credentialsId: 'HSM_USER_PIN', variable: 'PW')]) { + sh '''export HSM_USER_PIN=${PW}; node buildSrc/signDesktopClients.js''' + } - script { // upload to nexus - def util = load "ci/jenkins-lib/util.groovy" + uploadMacArtifacts() + } + } // stage mac + stage('Linux') { + when { expression { return params.LINUX } } + steps { + dir('build') { + unstash 'linux_installer' + unstash 'linux_installer_test' + } - util.publishToNexus( - groupId: "app", - artifactId: "desktop-linux-test", - version: "${VERSION}", - assetFilePath: "${WORKSPACE}/build/desktop-test/tutanota-desktop-test-linux.AppImage", - fileExtension: 'AppImage' - ) - util.publishToNexus( - groupId: "app", - artifactId: "desktop-win-test", - version: "${VERSION}", - assetFilePath: "${WORKSPACE}/build/desktop-test/tutanota-desktop-test-win.exe", - fileExtension: 'exe' - ) - util.publishToNexus( - groupId: "app", - artifactId: "desktop-mac-test", - version: "${VERSION}", - assetFilePath: "${WORKSPACE}/build/desktop-test/tutanota-desktop-test-mac.dmg", - fileExtension: 'dmg' - ) - util.publishToNexus( - groupId: "app", - artifactId: "desktop-linux", - version: "${VERSION}", - assetFilePath: "${WORKSPACE}/build/desktop/tutanota-desktop-linux.AppImage", - fileExtension: 'AppImage' - ) - util.publishToNexus( - groupId: "app", - artifactId: "desktop-win", - version: "${VERSION}", - assetFilePath: "${WORKSPACE}/build/desktop/tutanota-desktop-win.exe", - fileExtension: 'exe' - ) - util.publishToNexus( - groupId: "app", - artifactId: "desktop-mac", - version: "${VERSION}", - assetFilePath: "${WORKSPACE}/build/desktop/tutanota-desktop-mac.dmg", - fileExtension: 'dmg' - ) - } // script upload to nexus + withCredentials([string(credentialsId: 'HSM_USER_PIN', variable: 'PW')]) { + sh '''export HSM_USER_PIN=${PW}; node buildSrc/signDesktopClients.js''' + } - } // steps - } // stage build deb & publish + uploadLinuxArtifacts() + } + } // stage linux + } // parallel + } // stage sign and upload + } // stages + } // stage sign clients and upload to Nexus } // stages } // pipeline -void initBuildArea() { +def uploadWindowsArtifacts() { + script { + def artifactsMap = load "ci/jenkins-lib/desktop-artifacts-map.groovy" + def windowsFiles = artifactsMap.filesPathAndExt().windows + + uploadArtifacts("desktop-win-test", windowsFiles.staging) + uploadArtifacts("desktop-win", windowsFiles.prod) + } +} + +def uploadMacArtifacts() { + script { + def artifactsMap = load "ci/jenkins-lib/desktop-artifacts-map.groovy" + def macFiles = artifactsMap.filesPathAndExt().mac + + uploadArtifacts("desktop-mac-test", macFiles.staging) + uploadArtifacts("desktop-mac", macFiles.prod) + } +} + +def uploadLinuxArtifacts() { + script { + def artifactsMap = load "ci/jenkins-lib/desktop-artifacts-map.groovy" + def linuxFiles = artifactsMap.filesPathAndExt().linux + + uploadArtifacts("desktop-linux-test", linuxFiles.staging) + uploadArtifacts("desktop-linux", linuxFiles.prod) + } +} + +def uploadArtifacts(artifactId, filesPathAndExt) { + def util = load "ci/jenkins-lib/util.groovy" + + for (String[] file in filesPathAndExt) { + if (!fileExists(file[0])) { + currentBuild.result = 'ABORTED' + error("Unable to find file ${file[0]}") + } + + util.publishToNexus( + groupId: "app", + artifactId: artifactId, + version: "${VERSION}", + assetFilePath: "${WORKSPACE}/${file[0]}", + fileExtension: file[1] + ) + } +} + +def initBuildArea() { sh 'node -v' sh 'npm -v' sh 'npm ci' diff --git a/ci/Ios.Jenkinsfile b/ci/Ios.Jenkinsfile index 3f3ec3c1a47d..36c1feca7f9b 100644 --- a/ci/Ios.Jenkinsfile +++ b/ci/Ios.Jenkinsfile @@ -2,7 +2,6 @@ pipeline { environment { NODE_MAC_PATH = "/usr/local/opt/node@20/bin/" VERSION = sh(returnStdout: true, script: "${env.NODE_PATH}/node -p -e \"require('./package.json').version\" | tr -d \"\n\"") - RELEASE_NOTES_PATH = "app-ios/fastlane/metadata/default/release_notes.txt" } agent { @@ -11,10 +10,9 @@ pipeline { parameters { booleanParam( - name: 'RELEASE', + name: 'UPLOAD', defaultValue: false, - description: "Build testing and production version, and upload them to nexus/testflight/appstore. " + - "The production version will need to be released manually from appstoreconnect.apple.com." + description: "Upload staging/prod to Nexus" ) booleanParam( name: 'PROD', @@ -24,14 +22,20 @@ pipeline { name: 'STAGING', defaultValue: true ) - persistentText( - name: "releaseNotes", - defaultValue: "", - description: "release notes for this build" - ) } stages { + stage("Checking params") { + steps { + script{ + if(!params.STAGING && !params.PROD) { + currentBuild.result = 'ABORTED' + error('No artifacts were selected.') + } + } + echo "Params OKAY" + } + } // stage checking params stage('Check Github') { steps { script { @@ -42,7 +46,7 @@ pipeline { } stage("Run tests") { agent { - label 'mac-intel' + label 'mac' } environment { LC_ALL = "en_US.UTF-8" @@ -57,23 +61,21 @@ pipeline { sh 'fastlane test' } } - } - } + } // steps + } // stage run tests - stage("Build and upload to Apple") { + stage("Build") { environment { PATH="${env.NODE_MAC_PATH}:${env.PATH}:${env.HOME}/emsdk:${env.HOME}/emsdk/upstream/emscripten:${env.HOME}/emsdk/upstream/bin" MATCH_GIT_URL = "git@gitlab:/tuta/apple-certificates.git" LC_ALL = "en_US.UTF-8" LANG = "en_US.UTF-8" } - agent { - label 'mac-intel' - } - stages { + parallel { stage('Staging') { - when { - expression { return params.STAGING } + when { expression { return params.STAGING } } + agent { + label 'mac' } steps { script { @@ -81,16 +83,18 @@ pipeline { buildWebapp("test") generateXCodeProjects() util.runFastlane("de.tutao.tutanota.test", "adhoc_staging") - if (params.RELEASE) { - util.runFastlane("de.tutao.tutanota.test", "testflight_staging") + if (params.UPLOAD) { + util.runFastlane("de.tutao.tutanota.test", "build_mail_staging") + stash includes: "app-ios/releases/tutanota-${VERSION}-test.ipa", name: 'ipa-staging' } - stash includes: "app-ios/releases/tutanota-${VERSION}-adhoc-test.ipa", name: 'ipa-testing' + stash includes: "app-ios/releases/tutanota-${VERSION}-adhoc-test.ipa", name: 'ipa-adhoc-staging' } } - } + } // stage staging stage('Production') { - when { - expression { return params.PROD } + when { expression { return params.PROD } } + agent { + label 'mac' } steps { script { @@ -98,88 +102,64 @@ pipeline { buildWebapp("prod") generateXCodeProjects() util.runFastlane("de.tutao.tutanota", "adhoc_prod") - if (params.RELEASE) { - writeReleaseNotesForAppStore() - util.runFastlane("de.tutao.tutanota", "appstore_prod submit:true") + if (params.UPLOAD) { + util.runFastlane("de.tutao.tutanota", "build_mail_prod") + stash includes: "app-ios/releases/tutanota-${VERSION}.ipa", name: 'ipa-production' } - stash includes: "app-ios/releases/tutanota-${VERSION}-adhoc.ipa", name: 'ipa-production' + stash includes: "app-ios/releases/tutanota-${VERSION}-adhoc.ipa", name: 'ipa-adhoc-production' } - } - } - } - } + } // steps + } // stage production + } // stages + } // stage build stage('Upload to Nexus') { + when { expression { return params.UPLOAD } } environment { PATH = "${env.NODE_PATH}:${env.PATH}" } - when { - expression { return params.RELEASE } - } - agent { - label 'linux' - } - steps { - script { - if (params.STAGING) { - unstash 'ipa-testing' - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'There was an error when uploading to Nexus') { - publishToNexus("ios-test", "tutanota-${VERSION}-adhoc-test.ipa") - } + parallel { + stage("Staging") { + when { expression { return params.STAGING } } + agent { + label 'linux' } + steps { + unstash 'ipa-adhoc-staging' + unstash 'ipa-staging' - if (params.PROD) { - unstash 'ipa-production' - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'There was an error when uploading to Nexus') { - publishToNexus("ios", "tutanota-${VERSION}-adhoc.ipa") - } + uploadToNexus("ios-test", "tutanota-${VERSION}-adhoc-test.ipa", "adhoc.ipa") + uploadToNexus("ios-test", "tutanota-${VERSION}-test.ipa", "ipa") } } - } - } - - stage('Tag and create github release page') { - environment { - PATH = "${env.NODE_PATH}:${env.PATH}" - } - when { - expression { return params.RELEASE } - } - agent { - label 'linux' - } - steps { - script { - - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'Failed to create github release page for ios') { - def tag = "tutanota-ios-release-${VERSION}" - // need to run npm ci to install dependencies of releaseNotes.js - sh "npm ci" + stage("Production") { + when { expression { return params.PROD } } + agent { + label 'linux' + } + steps { + unstash 'ipa-adhoc-production' + unstash 'ipa-production' - writeFile file: "notes.txt", text: params.releaseNotes - withCredentials([string(credentialsId: 'github-access-token', variable: 'GITHUB_TOKEN')]) { - sh """node buildSrc/createReleaseDraft.js --name '${VERSION} (iOS)' \ - --tag 'tutanota-ios-release-${VERSION}' \ - --notes notes.txt""" - } // withCredentials - sh "rm notes.txt" - } // catchError + uploadToNexus("ios", "tutanota-${VERSION}-adhoc.ipa", "adhoc.ipa") + uploadToNexus("ios", "tutanota-${VERSION}.ipa", "ipa") + } } - } - } - } -} + } // parallel + } // stage upload to nexus + } // stages +} // pipeline -void stubClientDirectory() { +def stubClientDirectory() { script { sh "pwd" sh "echo $PATH" - sh "mkdir build-calendar-app" + sh "mkdir build-calendar-app" sh "mkdir build" } } -void buildWebapp(String stage) { +def buildWebapp(String stage) { script { sh "pwd" sh "echo $PATH" @@ -191,7 +171,7 @@ void buildWebapp(String stage) { } // Runs xcodegen on `projectPath`, a directory containing a `project.yml` -void generateXCodeProject(String projectPath, String spec) { +def generateXCodeProject(String projectPath, String spec) { // xcodegen ignores its --project and --project-roots flags // so we need to change the directory manually script { @@ -200,41 +180,22 @@ void generateXCodeProject(String projectPath, String spec) { } // Runs xcodegen on all of our project specs -void generateXCodeProjects() { +def generateXCodeProjects() { generateXCodeProject("app-ios", "mail-project") generateXCodeProject("tuta-sdk/ios", "project") } -void generateCalendarProject() { +def generateCalendarProject() { generateXCodeProject("app-ios", "calendar-project") } - -void writeReleaseNotesForAppStore() { - script { - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'Failed to create github release notes for ios') { - // need to run npm ci to install dependencies of releaseNotes.js - sh "npm ci" - writeFile file: "notes.txt", text: params.releaseNotes - withCredentials([string(credentialsId: 'github-access-token', variable: 'GITHUB_TOKEN')]) { - sh """node buildSrc/createReleaseDraft.js --name '${VERSION} (iOS)' \ - --tag 'tutanota-ios-release-${VERSION}'\ - --notes notes.txt \ - --toFile ${RELEASE_NOTES_PATH}""" - } - sh "rm notes.txt" - } - } - - sh "echo Created release notes for fastlane ${RELEASE_NOTES_PATH}" -} - -void publishToNexus(String artifactId, String ipaFileName) { +def uploadToNexus(String artifactId, String assetFileName, String fileExtension) { def util = load "ci/jenkins-lib/util.groovy" - util.publishToNexus(groupId: "app", + util.publishToNexus( + groupId: "app", artifactId: "${artifactId}", version: "${VERSION}", - assetFilePath: "${WORKSPACE}/app-ios/releases/${ipaFileName}", - fileExtension: "ipa" + assetFilePath: "${WORKSPACE}/app-ios/releases/${assetFileName}", + fileExtension: "${fileExtension}" ) } \ No newline at end of file diff --git a/ci/Publish-CalendarMobileArtifacts.Jenkinsfile b/ci/Publish-CalendarMobileArtifacts.Jenkinsfile index 99d2c2d33ca7..864bfab981c0 100644 --- a/ci/Publish-CalendarMobileArtifacts.Jenkinsfile +++ b/ci/Publish-CalendarMobileArtifacts.Jenkinsfile @@ -128,12 +128,12 @@ pipeline { } steps { script { - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'Failed to upload android app to GitHub') { + catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'Failed to upload iOS app to GitHub') { writeReleaseNotes("ios", "iOS", "${env.VERSION}", "") } } // script } // steps - } // stage Android App + } // stage iOS App } } stage("Publishing Artifacts to Stores") { diff --git a/ci/Publish-MailAndroidArtifacts.Jenkinsfile b/ci/Publish-MailAndroidArtifacts.Jenkinsfile new file mode 100644 index 000000000000..df1e42e4ed6f --- /dev/null +++ b/ci/Publish-MailAndroidArtifacts.Jenkinsfile @@ -0,0 +1,146 @@ +pipeline { + parameters { + string( + name: 'appVersion', + defaultValue: "", + description: 'Which version should be published' + ) + booleanParam( + name: 'STAGING', + defaultValue: true + description: "Uploads staging artifact (apk) to Google PlayStore" + ) + booleanParam( + name: 'PROD', + defaultValue: false + description: "Uploads production artifact (apk) to Google PlayStore as a Draft on the public track" + ) + booleanParam( + name: 'GITHUB_RELEASE', + defaultValue: false, + description: "Uploads android artifact (apk) to GitHub and publish release notes" + ) + persistentText( + name: 'releaseNotes', + defaultValue: "", + description: 'Android release notes' + ) + } + + environment { + PATH = "${env.NODE_PATH}:${env.PATH}" + PACKAGE_VERSION = sh(returnStdout: true, script: "${env.NODE_PATH}/node -p -e \"require('./package.json').version\" | tr -d \"\n\"") + VERSION = "${params.appVersion.trim() ?: PACKAGE_VERSION}" + GITHUB_RELEASE_PAGE = "https://github.com/tutao/tutanota/releases/tag/tutanota-android-release-${VERSION}" + PROD_FILE_PATH = "build/app-android/tutanota-app-tutao-release-${VERSION}.apk" + } + + agent { + label 'linux' + } + + stages { + stage("Checking params") { + steps { + script{ + if(!params.STAGING && !params.PROD && !params.GITHUB_RELEASE) { + currentBuild.result = 'ABORTED' + error('No tasks were selected.') + } + } + echo "Params OKAY" + } + } + stage("Google Play Store") { + stages { + stage("Staging") { + when { expression { return params.STAGING } } + steps { + script { + def filePath = "build/app-android/tutanota-app-tutao-releaseTest-${VERSION}.apk" + + downloadAndroidApp("android-test", filePath) + + // This doesn't publish to the main app on play store, + // instead it gets published to the hidden "tutanota-test" app + // this happens because the AppId is set to de.tutao.tutanota.test by the android build + // and play store knows which app to publish just based on the id + androidApkUpload( + googleCredentialsId: 'android-app-publisher-credentials', + apkFilesPattern: filePath, + trackName: 'internal', + rolloutPercentage: '100%', + recentChangeList: [ + [ + language: "en-US", + text : "see: ${GITHUB_RELEASE_PAGE}" + ] + ] + ) // androidApkUpload + } // script + } // steps + } // stage Testing + stage("Production") { + when { expression { return params.PROD } } + steps { + script { + downloadAndroidApp("android", PROD_FILE_PATH) + + androidApkUpload( + googleCredentialsId: 'android-app-publisher-credentials', + apkFilesPattern: "${PROD_FILE_PATH}", + trackName: 'production', + // Don't publish the app to users directly + // It will require manual intervention at play.google.com/console + rolloutPercentage: '0%', + recentChangeList: [ + [ + language: "en-US", + text : "see: ${GITHUB_RELEASE_PAGE}" + ] + ] + ) + } // script + } // steps + } // stage Production + } // stages + } // stage Google Play Store + stage("Github release notes") { + when { expression { return params.PROD && params.GITHUB_RELEASE } } + steps { + script { + downloadAndroidApp("android", PROD_FILE_PATH) + + sh 'npm ci' + + writeFile file: "notes.txt", text: params.releaseNotes + withCredentials([string(credentialsId: 'github-access-token', variable: 'GITHUB_TOKEN')]) { + sh """node buildSrc/createReleaseDraft.js --name '${VERSION} (Android)' \ + --tag 'tutanota-android-release-${VERSION}' \ + --uploadFile '${WORKSPACE}/${PROD_FILE_PATH}' \ + --notes notes.txt""" + } // withCredentials + sh "rm notes.txt" + } // script + } // steps + } // stage github release notes + } // stages +} // pipeline + +def downloadAndroidApp(String artifactId, String filePath) { + def util = load "ci/jenkins-lib/util.groovy" + + util.downloadFromNexus( + groupId: "app", + artifactId: artifactId, + version: "${VERSION}", + outFile: "${WORKSPACE}/${filePath}", + fileExtension: 'apk' + ) + + if (!fileExists("${filePath}")) { + currentBuild.result = 'ABORTED' + error("Unable to find file ${filePath}") + } + echo "File ${filePath} found!" +} \ No newline at end of file diff --git a/ci/Publish-MailDesktopArtifacts.Jenkinsfile b/ci/Publish-MailDesktopArtifacts.Jenkinsfile new file mode 100644 index 000000000000..2f97b369e1fb --- /dev/null +++ b/ci/Publish-MailDesktopArtifacts.Jenkinsfile @@ -0,0 +1,192 @@ +pipeline { + environment { + // on m1 macs, this is a symlink that must be updated. see wiki. + VERSION = sh(returnStdout: true, script: "${env.NODE_PATH}/node -p -e \"require('./package.json').version\" | tr -d \"\n\"") + TMPDIR ='/tmp' + } + + parameters { + booleanParam( + name: 'DEB', + defaultValue: true, + description: "build deb package" + ) + booleanParam( + name: 'GITHUB_RELEASE', + defaultValue: false, + description: "publish release notes draft" + ) + persistentText( + name: 'releaseNotes', + defaultValue: '', + description: "release notes for this build" + ) + } + + agent { + label 'master' + } + + stages { + stage("Checking params") { + steps { + script{ + if(!params.STAGING && !params.PROD && !params.GITHUB_RELEASE) { + currentBuild.result = 'ABORTED' + error('No tasks were selected.') + } + } + echo "Params OKAY" + } + } + stage('Check Github') { + steps { + script { + def util = load "ci/jenkins-lib/util.groovy" + util.checkGithub() + } + } + } + stage('Preparation for build deb and publish notes') { + agent { + label 'master' + } + steps { + script { + def devicePath = sh(script: 'lsusb | grep Nitro | sed -nr \'s|Bus (.*) Device ([^:]*):.*|/dev/bus/usb/\\1/\\2|p\'', returnStdout: true).trim() + env.DEVICE_PATH = devicePath + } + } + } + stage ('Build deb and publish release notes draft') { + when { expression { return params.GITHUB_RELEASE || params.DEB } } + agent { + dockerfile { + filename 'linux-build.dockerfile' + label 'master' + dir 'ci/containers' + additionalBuildArgs '--format docker' + args "--network host -v /run:/run:rw,z -v /opt/repository:/opt/repository:rw,z --device=${env.DEVICE_PATH}" + } // docker + } // agent + stages { + stage('Build deb') { + when { expression { return params.DEB } } + steps { + downloadWindowArtifacts() + downloadMacArtifacts() + downloadLinuxArtifacts() + + sh 'node -v' + sh 'npm -v' + sh 'npm ci' + sh 'npm run build-packages' + sh 'node buildSrc/publish.js desktop' + } // steps + } // stage build deb + stage('Publish release notes draft') { + when { expression { return params.GITHUB_RELEASE } } + steps { + script { + def desktopLinux = "build/desktop/tutanota-desktop-linux.AppImage" + def desktopWin = "build/desktop/tutanota-desktop-win.exe" + def desktopMac = "build/desktop/tutanota-desktop-mac.dmg" + + def util = load "ci/jenkins-lib/util.groovy" + + util.downloadFromNexus( groupId: "app", + artifactId: "desktop-win", + version: "${VERSION}", + outFile: "${WORKSPACE}/${desktopWin}", + fileExtension: 'exe') + if (!fileExists("${desktopWin}")) { + currentBuild.result = 'ABORTED' + error("Unable to find file ${desktopWin}") + } + + util.downloadFromNexus( groupId: "app", + artifactId: "desktop-mac", + version: "${VERSION}", + outFile: "${WORKSPACE}/${desktopMac}", + fileExtension: 'dmg') + if (!fileExists("${desktopMac}")) { + currentBuild.result = 'ABORTED' + error("Unable to find file ${desktopMac}") + } + + util.downloadFromNexus( groupId: "app", + artifactId: "desktop-linux", + version: "${VERSION}", + outFile: "${WORKSPACE}/${desktopLinux}", + fileExtension: 'AppImage') + if (!fileExists("${desktopLinux}")) { + currentBuild.result = 'ABORTED' + error("Unable to find file ${desktopLinux}") + } + + writeFile file: "notes.txt", text: params.releaseNotes + withCredentials([string(credentialsId: 'github-access-token', variable: 'GITHUB_TOKEN')]) { + sh """node buildSrc/createReleaseDraft.js --name '${VERSION} (Desktop)' \ + --tag 'tutanota-desktop-release-${VERSION}' \ + --uploadFile '${WORKSPACE}/${desktopLinux}' \ + --uploadFile '${WORKSPACE}/${desktopWin}' \ + --uploadFile '${WORKSPACE}/${desktopMac}' \ + --notes notes.txt""" + } // withCredentials + sh "rm notes.txt" + } // script release draft + } // steps + } // stage publish release notes draft + } // stages + } // stage build deb and publish release notes + } // stages +} // pipeline + +def downloadWindowArtifacts() { + script { + def artifactsMap = load "ci/jenkins-lib/desktop-artifacts-map.groovy" + def windowsFiles = artifactsMap.filesPathAndExt().windows + + downloadArtifacts("desktop-win-test", windowsFiles.staging) + downloadArtifacts("desktop-win", windowsFiles.prod) + } +} + +def downloadMacArtifacts() { + script { + def artifactsMap = load "ci/jenkins-lib/desktop-artifacts-map.groovy" + def macFiles = artifactsMap.filesPathAndExt().mac + + downloadArtifacts("desktop-mac-test", macFiles.staging) + downloadArtifacts("desktop-mac", macFiles.prod) + } +} + +def downloadLinuxArtifacts() { + script { + def artifactsMap = load "ci/jenkins-lib/desktop-artifacts-map.groovy" + def linuxFiles = artifactsMap.filesPathAndExt().linux + + downloadArtifacts("desktop-linux-test", linuxFiles.staging) + downloadArtifacts("desktop-linux", linuxFiles.prod) + } +} + +def downloadArtifacts(artifactId, filesPathAndExt) { + def util = load "ci/jenkins-lib/util.groovy" + + for (String[] file in filesPathAndExt) { + util.downloadFromNexus( + groupId: "app", + artifactId: artifactId, + version: "${VERSION}", + outFile: "${WORKSPACE}/${file[0]}", + fileExtension: file[1] + ) + + if (!fileExists(file[0])) { + currentBuild.result = 'ABORTED' + error("Unable to find file ${file[0]}") + } + } +} \ No newline at end of file diff --git a/ci/Publish-MailiOSArtifacts.Jenkinsfile b/ci/Publish-MailiOSArtifacts.Jenkinsfile new file mode 100644 index 000000000000..853b98cf265a --- /dev/null +++ b/ci/Publish-MailiOSArtifacts.Jenkinsfile @@ -0,0 +1,164 @@ +pipeline { + parameters { + string( + name: 'appVersion', + defaultValue: "", + description: 'Which version should be published' + ) + booleanParam( + name: 'STAGING', + defaultValue: true + ) + booleanParam( + name: 'PROD', + defaultValue: false + ) + booleanParam( + name: 'APP_STORE_NOTES', + defaultValue: false, + description: "Publish iOS release notes to the Apple App Store" + ) + booleanParam( + name: 'GITHUB_RELEASE', + defaultValue: false, + description: "Publish iOS release notes to Github" + ) + persistentText( + name: 'releaseNotes', + defaultValue: "", + description: "iOS release notes" + ) + } + + environment { + PATH="${env.NODE_PATH}:${env.PATH}" + PACKAGE_VERSION = sh(returnStdout: true, script: "${env.NODE_PATH}/node -p -e \"require('./package.json').version\" | tr -d \"\n\"") + VERSION = "${params.appVersion.trim() ?: PACKAGE_VERSION}" + GITHUB_RELEASE_PAGE = "https://github.com/tutao/tutanota/releases/tag/tutanota-ios-release-${VERSION}" + FILE_PATH_STAGING = "build/app-ios/releases/tutanota-${VERSION}-test.ipa" + FILE_PATH_PROD = "build/app-ios/releases/tutanota-${VERSION}.ipa" + } + + agent { + label 'linux' + } + + stages { + stage("Checking params") { + steps { + script{ + if(!params.STAGING && !params.PROD && !params.GITHUB_RELEASE) { + currentBuild.result = 'ABORTED' + error('No tasks were selected.') + } + } + echo "Params OKAY" + } + } + stage("Download artifacts") { + parallel { + stage("Staging") { + when { expression { return params.STAGING } } + steps { + downloadIOSApp("ios-test", FILE_PATH_STAGING) + stash includes: FILE_PATH_STAGING, name: 'ipa-staging' + } + } + stage("Production") { + when { expression { return params.PROD } } + steps { + downloadIOSApp("ios", FILE_PATH_PROD) + stash includes: FILE_PATH_PROD, name: 'ipa-prod' + } + } + } // parallel + } // stage download artifacts + stage("Apple App Store") { + when { expression { return params.STAGING || params.PROD } } + environment { + MAC_NODE_PATH = "/usr/local/opt/node@20/bin/" + PATH="/opt/homebrew/bin:${/usr/local/opt/node@20/bin/}:${env.PATH}" + MATCH_GIT_URL = "git@gitlab:/tuta/apple-certificates.git" + LC_ALL = "en_US.UTF-8" + LANG = "en_US.UTF-8" + } + agent { + label 'mac' + } + stages { + stage("Staging") { + when { expression { return params.STAGING } } + steps { + unstash "ipa-staging" + + script { + def util = load "ci/jenkins-lib/util.groovy" + util.runFastlane("de.tutao.tutanota.test", "publish_mail_staging file:${WORKSPACE}/${FILE_PATH_STAGING}") + } + } // steps + } // stage staging + stage("Production") { + when { expression { return params.PROD } } + environment { + RELEASE_NOTES_PATH = "app-ios/fastlane/metadata/default/release_notes.txt" + } + steps { + unstash "ipa-prod" + + script { + if (params.APP_STORE_NOTES) { + // need to run npm ci to install dependencies of releaseNotes.js + sh "npm ci" + + writeFile file: "notes.txt", text: params.releaseNotes + sh """node buildSrc/createReleaseDraft.js --name '${VERSION} (iOS)' \ + --tag 'tutanota-ios-release-${VERSION}'\ + --notes notes.txt \ + --toFile ${RELEASE_NOTES_PATH}""" + sh "rm notes.txt" + sh "echo Created release notes for fastlane ${RELEASE_NOTES_PATH}" + } + + def util = load "ci/jenkins-lib/util.groovy" + util.runFastlane("de.tutao.tutanota", "publish_mail_prod file:${WORKSPACE}/${FILE_PATH_PROD}") + } + } // steps + } // stage production + } // stages + } // stage apple app store + stage("GitHub release notes") { + when { expression { return params.GITHUB_RELEASE } } + steps { + script { + sh 'npm ci' + + writeFile file: "notes.txt", text: params.releaseNotes + withCredentials([string(credentialsId: 'github-access-token', variable: 'GITHUB_TOKEN')]) { + sh """node buildSrc/createReleaseDraft.js --name '${VERSION} (iOS)' \ + --tag 'tutanota-ios-release-${VERSION}' \ + --notes notes.txt""" + } // withCredentials + sh "rm notes.txt" + } // script + } // steps + } // stage GitHub release notes + } // stages +} // pipeline + +def downloadIOSApp(String artifactId, String filePath) { + def util = load "ci/jenkins-lib/util.groovy" + + util.downloadFromNexus( + groupId: "app", + artifactId: "${artifactId}", + version: "${VERSION}", + outFile: "${WORKSPACE}/${filePath}", + fileExtension: "ipa" + ) + + if (!fileExists("${filePath}")) { + currentBuild.result = 'ABORTED' + error("Unable to find file ${filePath}") + } + echo "File ${filePath} found!" +} \ No newline at end of file diff --git a/ci/Publish-WebappArtifacts.Jenkinsfile b/ci/Publish-WebappArtifacts.Jenkinsfile new file mode 100644 index 000000000000..3e7683f69c79 --- /dev/null +++ b/ci/Publish-WebappArtifacts.Jenkinsfile @@ -0,0 +1,105 @@ +pipeline { + environment { + PATH="${env.NODE_PATH}:${env.PATH}" + VERSION=sh(returnStdout: true, script: "${env.NODE_PATH}/node -p -e \"require('./package.json').version\" | tr -d \"\n\"") + } + options { + preserveStashes() + } + parameters { + booleanParam( + name: 'DEB', + defaultValue: true, + description: "Build deb package" + ) + booleanParam( + name: 'PUBLISH_NPM_MODULES', + defaultValue: false, + description: "Publish npm modules" + ) + booleanParam( + name: 'GITHUB_RELEASE', + defaultValue: false, + description: "Publish release notes draft" + ) + persistentText( + name: 'releaseNotes', + defaultValue: '', + description: "Release notes for this build" + ) + } + agent { + label 'linux' + } + stages { + stage("Checking params") { + steps { + script{ + if(!params.DEB && !params.PUBLISH_NPM_MODULES && !params.GITHUB_RELEASE) { + currentBuild.result = 'ABORTED' + error('No tasks were selected.') + } + } + echo "Params OKAY" + } + } // stage checking params + stage('Check Github') { + steps { + script { + def util = load "ci/jenkins-lib/util.groovy" + util.checkGithub() + } + } + } + stage('Build deb') { + when { expression { return params.DEB } } + steps { + script { + def filePath = "webapp_built.tar.gz" + def util = load "ci/jenkins-lib/util.groovy" + + + util.downloadFromNexus( + groupId: "app", + artifactId: "webapp", + version: "${VERSION}", + outFile: "${WORKSPACE}/${filePath}", + fileExtension: "tar.gz" + ) + if (!fileExists(filePath)) { + currentBuild.result = 'ABORTED' + error("Unable to find file ${filePath}") + } + } + + sh "tar -xvzf ${filePath}" + sh 'npm ci' + sh 'node buildSrc/publish.js webapp' + } // steps + } // stage build deb + + stage('Publish release notes') { + when { expression { return params.GITHUB_RELEASE } } + steps { + sh 'npm ci' + + writeFile file: "notes.txt", text: params.releaseNotes + withCredentials([string(credentialsId: 'github-access-token', variable: 'GITHUB_TOKEN')]) { + sh '''node buildSrc/createReleaseDraft.js --name ${VERSION} --tag tutanota-release-${VERSION} --notes notes.txt''' + } + } + } // stage publish release notes + + stage('Publish npm modules') { + when { expression { return params.PUBLISH_NPM_MODULES } } + steps { + sh 'npm ci' + sh 'npm run build-packages' + // .npmrc expects $NPM_TOKEN + withCredentials([string(credentialsId: 'npm-token',variable: 'NPM_TOKEN')]) { + sh "npm --workspaces publish --access public" + } + } // steps + } // stage publish npm modules + } // stages +} // pipeline diff --git a/ci/Release.Jenkinsfile b/ci/Release.Jenkinsfile index 391cb157bb80..6e6a3a8e34c2 100644 --- a/ci/Release.Jenkinsfile +++ b/ci/Release.Jenkinsfile @@ -9,20 +9,20 @@ pipeline { } parameters { - booleanParam( - name: 'dryRun', - defaultValue: true, - description: "builds the clients and generates release notes if requested, but doesn't modify github or upload any artifacts." - ) + choice( + name: 'target', + choices: ['dryRun', 'buildAndPublishToStaging', 'publishToStaging', 'publishToProd'], + description: "publishToStaging and publishToProd do not build. They download artifacts from nexus" + ) booleanParam( name: 'generateReleaseNotes', defaultValue: false, - description: "check if the release notes should be updated in the downstream jobs, uncheck if last runs release notes should be reused." + description: "Check if the release notes should be updated in the downstream jobs, uncheck if last runs release notes should be reused" ) persistentString( name: 'milestone', defaultValue: '', - description: 'Which github milestone to reference for generating release notes. leave empty to use version number.' + description: 'Which github milestone to reference for generating release notes. leave empty to use version number' ) booleanParam( name: "dictionaries", @@ -32,7 +32,7 @@ pipeline { booleanParam( name: 'web', defaultValue: true, - description: "Build the web app and packages (required to build the other clients if the version changed)." + description: "Build the web app and packages (required to build the other clients if the version changed)" ) booleanParam( name: 'ios', @@ -70,73 +70,166 @@ pipeline { releaseNotes = reviewReleaseNotes(web, android, desktop, ios, version) echo("${releaseNotes}") - } // script release notes + } // script } // steps } // stage prepare release notes - stage("web app & packages") { - when { expression { return params.web } } - agent { label 'master'} - steps { - build job: 'tutanota-3-webapp', parameters: params.generateReleaseNotes ? [ - booleanParam(name: 'RELEASE', value: !params.dryRun), - text(name: "releaseNotes", value: releaseNotes.web), - ] : [ booleanParam(name: "RELEASE", value: !params.dryRun) ] - } // steps - } // stage web app & packages - stage("other clients") { + stage("Clients") { + environment { + BUILD = "${params.target.equals("dryRun") || params.target.equals("buildAndPublishToStaging")}" + PUBLISH_STAGING = "${params.target.equals("buildAndPublishToStaging") || params.target.equals("publishToStaging")}" + } parallel { + stage("Web App & Packages") { + when { expression { return params.web } } + agent { label 'master'} + stages { + stage("Build Web") { + when { expression { return BUILD.toBoolean() } } + steps { + build job: 'tutanota-3-webapp', parameters: [ + booleanParam(name: "UPLOAD", value: params.target.equals("buildAndPublishToStaging")) + ] + } // steps + } // stage build + stage("Publish Web") { + when { expression { return !params.target.equals("dryRun") } } + steps { + build job: 'tutanota-3-webapp-publish', parameters: params.generateReleaseNotes ? [ + booleanParam(name: 'DEB', value: true), + booleanParam(name: 'PUBLISH_NPM_MODULES', value: params.target.equals("buildAndPublishToStaging")), + booleanParam(name: 'GITHUB_RELEASE', value: params.target.equals("publishToProd")), + text(name: "releaseNotes", value: releaseNotes.web), + ] : [ + booleanParam(name: 'DEB', value: true), + booleanParam(name: 'PUBLISH_NPM_MODULES', value: params.target.equals("buildAndPublishToStaging")), + booleanParam(name: 'GITHUB_RELEASE', value: false), + ] + } // steps + } // stage publish + } // stages + } // stage web app & packages + stage("Desktop Dicts") { when { expression { return params.dictionaries } } steps { script { - build job: 'tutanota-3-desktop-dictionaries', parameters: [booleanParam(name: "RELEASE", value: !params.dryRun)] - } // script - } - } - stage("Desktop Client") { - when { expression { return params.desktop } } - steps { - script { - build job: 'tutanota-3-desktop', parameters: params.generateReleaseNotes ? [ - booleanParam(name: "RELEASE", value: !params.dryRun), - text(name: "releaseNotes", value: releaseNotes.desktop), - ] : [ - booleanParam(name: "RELEASE", value: !params.dryRun), + build job: 'tutanota-3-desktop-dictionaries', parameters: [ + booleanParam(name: "RELEASE", value: !params.target.equals("dryRun")) ] } // script } // steps + } // stage desktop dicts + + stage("Desktop Client") { + when { expression { return params.desktop } } + stages { + stage("Build Desktop") { + when { expression { return BUILD.toBoolean() } } + steps { + script { + build job: 'tutanota-3-desktop', parameters: [ + booleanParam(name: "UPLOAD", value: params.target.equals("buildAndPublishToStaging")), + booleanParam(name: "WINDOWS", value: true), + booleanParam(name: "MAC", value: true), + booleanParam(name: "LINUX", value: true), + ] + } // script + } // steps + } // stage build + stage("Publish Desktop") { + when { expression { return !params.target.equals("dryRun") } } + steps { + script { + build job: 'tutanota-3-desktop-publish', parameters: params.generateReleaseNotes ? [ + booleanParam(name: "DEB", value: true), + booleanParam(name: "GITHUB_RELEASE", value: params.target.equals("publishToProd")), + text(name: "releaseNotes", value: releaseNotes.desktop), + ] : [ + booleanParam(name: "DEB", value: true), + booleanParam(name: "GITHUB_RELEASE", value: false), + ] + } // script + } // steps + } // stage publish + } // stages } // stage desktop client + stage("iOS Client") { when { expression { return params.ios } } - steps { - script { - build job: 'tutanota-3-ios', parameters: params.generateReleaseNotes ? [ - booleanParam(name: "RELEASE", value: !params.dryRun), - text(name: "releaseNotes", value: releaseNotes.ios), - booleanParam(name: "STAGING", value: true), - booleanParam(name: "PROD", value: true), - ] : [ - booleanParam(name: "RELEASE", value: !params.dryRun), - booleanParam(name: "STAGING", value: true), - booleanParam(name: "PROD", value: true), - ] - } // script - } // steps - } // stage desktop client + stages { + stage("Build iOS") { + when { expression { return BUILD.toBoolean() } } + steps { + script { + build job: 'tutanota-3-ios', parameters: [ + booleanParam(name: "UPLOAD", value: params.target.equals("buildAndPublishToStaging")), + booleanParam(name: "STAGING", value: params.target.equals("buildAndPublishToStaging")), + booleanParam(name: "PROD", value: true), + ] + } // script + } // steps + } // stage build + stage("Publish iOS") { + when { expression { return !params.target.equals("dryRun") } } + steps { + script { + build job: 'tutanota-3-ios-publish', parameters: params.generateReleaseNotes ? [ + text(name: "appVersion", value: params.milestone), + text(name: "STAGING", PUBLISH_STAGING.toBoolean()), + text(name: "PROD", value: params.target.equals("publishToProd")), + text(name: "APP_STORE_NOTES", value: params.target.equals("publishToProd")), + text(name: "GITHUB_RELEASE", value: params.target.equals("publishToProd")), + text(name: "releaseNotes", value: releaseNotes.ios), + ] : [ + text(name: "appVersion", value: params.milestone), + text(name: "STAGING", PUBLISH_STAGING.toBoolean()), + text(name: "PROD", value: params.target.equals("publishToProd")), + text(name: "APP_STORE_NOTES", value: false), + text(name: "GITHUB_RELEASE", value: false), + ] + } // script + } // steps + } // stage publish + } // stages + } // stage ios client + stage("Android Client") { when { expression { return params.android } } - steps { - script { - build job: 'tutanota-3-android', parameters: params.generateReleaseNotes ? [ - booleanParam(name: "RELEASE", value: !params.dryRun), - text(name: "releaseNotes", value: releaseNotes.android), - ] : [ - booleanParam(name: "RELEASE", value: !params.dryRun), - ] - } // script - } // steps - } // stage desktop client - } // parallel clients + stages { + stage("Build Android") { + when { expression { return BUILD.toBoolean() } } + steps { + script { + build job: 'tutanota-3-android', parameters: [ + booleanParam(name: "UPLOAD", value: params.target.equals("buildAndPublishToStaging")), + booleanParam(name: "STAGING", value: params.target.equals("buildAndPublishToStaging")), + booleanParam(name: "PROD", value: true), + ] + } // script + } // steps + } // stage build + stage("Publish Android") { + when { expression { return !params.target.equals("dryRun") } } + steps { + script { + build job: 'tutanota-3-android-publish', parameters: params.generateReleaseNotes ? [ + text(name: "appVersion", value: params.milestone), + text(name: "STAGING", PUBLISH_STAGING.toBoolean()), + text(name: "PROD", value: params.target.equals("publishToProd")), + text(name: "GITHUB_RELEASE", value: params.target.equals("publishToProd")), + text(name: "releaseNotes", value: releaseNotes.android), + ] : [ + text(name: "appVersion", value: params.milestone), + text(name: "STAGING", PUBLISH_STAGING.toBoolean()), + text(name: "PROD", value: params.target.equals("publishToProd")), + text(name: "GITHUB_RELEASE", value: false), + ] + } // script + } // steps + } // stage publish + } // stages + } // stage android client + } // parallel } // stage other clients } // stages } // pipeline diff --git a/ci/Webapp.Jenkinsfile b/ci/Webapp.Jenkinsfile index 2f840c870d13..584fabff2c9a 100644 --- a/ci/Webapp.Jenkinsfile +++ b/ci/Webapp.Jenkinsfile @@ -1,21 +1,16 @@ pipeline { environment { - PATH="${env.NODE_PATH}:${env.PATH}:/home/jenkins/emsdk/upstream/bin/:/home/jenkins/emsdk/:/home/jenkins/emsdk/upstream/emscripten" + PATH="${env.NODE_PATH}:${env.PATH}:/home/jenkins/emsdk/upstream/bin/:/home/jenkins/emsdk/:/home/jenkins/emsdk/upstream/emscripten" } options { preserveStashes() } parameters { booleanParam( - name: 'RELEASE', + name: 'UPLOAD', defaultValue: false, - description: "Prepare a release version (doesn't publish to production, this is done manually). Also publishes NPM modules" + description: "Upload release version to Nexus" ) - persistentText( - name: "releaseNotes", - defaultValue: "", - description: "release notes for this build" - ) } agent { label 'master' @@ -37,8 +32,13 @@ pipeline { sh 'npm ci' sh 'npm run build-packages' sh 'node webapp.js release' - // excluding web-specific and mobile specific parts which we don't need in desktop - stash includes: 'build/**', excludes: '**/app.html, **/desktop.html, **/index-app.js, **/index-desktop.js', name: 'webapp_built' + + script { + if (params.UPLOAD) { + // excluding web-specific and mobile specific parts which we don't need in desktop + stash includes: 'build/**', excludes: '**/app.html, **/desktop.html, **/index-app.js, **/index-desktop.js', name: 'webapp_built' + } + } // Bundle size stats publishHTML target: [ @@ -61,51 +61,36 @@ pipeline { reportName: 'bundle dependencies' ] } - } + } // stage build - stage('Publish') { + stage('Upload to Nexus') { + when { + expression { return params.UPLOAD } + } environment { VERSION = sh(returnStdout: true, script: "node -p -e \"require('./package.json').version\" | tr -d \"\n\"") } - when { - expression { return params.RELEASE } - } agent { label 'linux' } steps { - sh 'echo Publishing version $VERSION' - sh 'npm ci' + sh 'echo Uploading version $VERSION' sh 'rm -rf ./build/*' - unstash 'webapp_built' - sh 'node buildSrc/publish.js webapp' - writeFile file: "notes.txt", text: params.releaseNotes + sh 'tar -cvzf webapp_built.tar.gz ./build' - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', message: 'Failed to create github release for webapp') { - withCredentials([string(credentialsId: 'github-access-token', variable: 'GITHUB_TOKEN')]) { - sh '''node buildSrc/createReleaseDraft.js --name ${VERSION} --tag tutanota-release-${VERSION} --notes notes.txt''' - } - } - } - } + script { + def util = load "ci/jenkins-lib/util.groovy" - stage('Publish npm modules') { - when { - expression { return params.RELEASE } - } - agent { - label 'linux' - } - steps { - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh "npm ci && npm run build-packages" - // .npmrc expects $NPM_TOKEN - withCredentials([string(credentialsId: 'npm-token',variable: 'NPM_TOKEN')]) { - sh "npm --workspaces publish --access public" - } - } - } - } - } -} + util.publishToNexus( + groupId: "app", + artifactId: "webapp", + version: "${VERSION}", + assetFilePath: "${WORKSPACE}/webapp_built.tar.gz", + fileExtension: "tar.gz" + ) + } + } // steps + } // stage upload to nexus + } // stages +} // pipeline diff --git a/ci/jenkins-lib/desktop-artifacts-map.groovy b/ci/jenkins-lib/desktop-artifacts-map.groovy new file mode 100644 index 000000000000..6461a83a10fc --- /dev/null +++ b/ci/jenkins-lib/desktop-artifacts-map.groovy @@ -0,0 +1,53 @@ +def filesPathAndExt() { + return [ + windows: [ + staging: [ + ["build/desktop-test/tutanota-desktop-test-win.exe", "exe"], + ["build/desktop-test/tutanota-desktop-test-win.exe.blockmap", "exe.blockmap"], + ["build/desktop-test/win-sig.bin", "sig.bin"], + ["build/desktop-test/latest.yml", "latest.yml"], + ], + prod : [ + ["build/desktop/tutanota-desktop-win.exe", "exe"], + ["build/desktop/tutanota-desktop-win.exe.blockmap", "exe.blockmap"], + ["build/desktop/win-sig.bin", "sig.bin"], + ["build/desktop/latest.yml", "latest.yml"], + ] + ], + mac : [ + staging: [ + ["build/desktop-test/tutanota-desktop-test-mac.dmg", "dmg"], + ["build/desktop-test/tutanota-desktop-test-mac.dmg.blockmap", "dmg.blockmap"], + ["build/desktop-test/tutanota-desktop-test-mac.zip", "zip"], + ["build/desktop-test/tutanota-desktop-test-mac.zip.blockmap", "zip.blockmap"], + ["build/desktop-test/mac-sig-dmg.bin", "sig.dmg.bin"], + ["build/desktop-test/mac-sig-zip.bin", "sig.zip.bin"], + ["build/desktop-test/latest-mac.yml", "latest.yml"], + ], + prod : [ + ["build/desktop/tutanota-desktop-mac.dmg", "dmg"], + ["build/desktop/tutanota-desktop-mac.dmg.blockmap", "dmg.blockmap"], + ["build/desktop/tutanota-desktop-mac.zip", "zip"], + ["build/desktop/tutanota-desktop-mac.zip.blockmap", "zip.blockmap"], + ["build/desktop/mac-sig-dmg.bin", "sig.dmg.bin"], + ["build/desktop/mac-sig-zip.bin", "sig.zip.bin"], + ["build/desktop/latest-mac.yml", "latest.yml"], + ] + ], + linux : [ + staging: [ + ["build/desktop-test/tutanota-desktop-test-linux.AppImage", "AppImage"], + ["build/desktop-test/linux-sig.bin", "sig.bin"], + ["build/desktop-test/latest-linux.yml", "latest.yml"], + ], + prod : [ + ["build/desktop/tutanota-desktop-linux.AppImage", "AppImage"], + ["build/desktop/linux-sig.bin", "sig.bin"], + ["build/desktop/latest-linux.yml", "latest.yml"], + ] + ], + ] +} + +// required in order to be able to use "load" to include this script in a jenkins pipleline +return this \ No newline at end of file diff --git a/ci/jenkins-lib/util.groovy b/ci/jenkins-lib/util.groovy index 7232a6c49bba..adafe3add235 100644 --- a/ci/jenkins-lib/util.groovy +++ b/ci/jenkins-lib/util.groovy @@ -7,7 +7,7 @@ def publishToNexus(Map params) { "-F maven2.groupId=${params.groupId} " + "-F maven2.artifactId=${params.artifactId} " + "-F maven2.version=${params.version} " + - "-F maven2.generate-pom=true " + + "-F maven2.generate-pom=false " + "-F maven2.asset1=@${params.assetFilePath} " + "-F maven2.asset1.extension=${params.fileExtension}" }