diff --git a/src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClient.groovy b/src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClient.groovy index 496907ca..b584bb88 100644 --- a/src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClient.groovy +++ b/src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClient.groovy @@ -3,6 +3,8 @@ package io.saagie.plugin.dataops.clients import groovy.json.JsonOutput import groovy.json.JsonSlurper import io.saagie.plugin.dataops.DataOpsExtension +import io.saagie.plugin.dataops.models.App +import io.saagie.plugin.dataops.models.AppVersionDTO import io.saagie.plugin.dataops.models.EnvVarScopeTypeEnum import io.saagie.plugin.dataops.models.ExportApp import io.saagie.plugin.dataops.models.ExportJob @@ -20,6 +22,7 @@ import io.saagie.plugin.dataops.tasks.projects.enums.JobV1Category import io.saagie.plugin.dataops.tasks.projects.enums.JobV1Type import io.saagie.plugin.dataops.tasks.service.TechnologyService import io.saagie.plugin.dataops.tasks.service.exportTask.ExportContainer +import io.saagie.plugin.dataops.tasks.service.importTask.ImportAppService import io.saagie.plugin.dataops.tasks.service.importTask.ImportJobService import io.saagie.plugin.dataops.tasks.service.importTask.ImportPipelineService import io.saagie.plugin.dataops.tasks.service.importTask.ImportVariableService @@ -78,12 +81,13 @@ class SaagieClient { this.checkBaseConfiguration() def allTechnologies = saagieUtils.&getListAllTechnologiesRequest def allTechnologyVersions = saagieUtils.&getListTechnologyVersionsRequest - + def allTechnologiesForApp = saagieUtils.&getAppTechnologiesList TechnologyService.instance.init( client, allTechnologies, allTechnologyVersions, - slurper + slurper, + allTechnologiesForApp ) } @@ -932,7 +936,7 @@ class SaagieClient { (exportVariables, variablesExportedIsEmpty) = getVariableListIfConfigIsDefined(this.&getListVariablesV1FromConfig) // We need to put null because we don't export applications from v1. - return export(exportPipelines, exportJobs, exportVariables,null, listJobsByNameAndIdFromV1, variablesExportedIsEmpty, true) + return export(exportPipelines, exportJobs, exportVariables, null, listJobsByNameAndIdFromV1, variablesExportedIsEmpty, true) } String exportArtifactsV2() { @@ -957,7 +961,7 @@ class SaagieClient { (exportPipelines, exportJobs, exportVariables, variablesExportedIsEmpty, exportApps) = exportArtifactsFromProjectConfiguration() } - return export(exportPipelines, exportJobs, exportVariables, exportApps, variablesExportedIsEmpty ) + return export(exportPipelines, exportJobs, exportVariables, exportApps, variablesExportedIsEmpty) } @@ -1076,12 +1080,12 @@ class SaagieClient { def parsedResult = slurper.parseText(responseBody) if (parsedResult.data == null || parsedResult.data.labWebApp == null) { def message = null - if(parsedResult.data == null) { + if (parsedResult.data == null) { message = "Something went wrong when getting app detail: $responseBody for app id $appId" logger.error(message) } - if(parsedResult.data.labWebApp == null) { + if (parsedResult.data.labWebApp == null) { message = "App with id $appId does not exist" } @@ -1831,12 +1835,16 @@ class SaagieClient { def jobsConfigFromExportedZip = SaagieClientUtils.extractJobConfigAndPackageFromExportedJob(exportedArtifactsPathRoot) def pipelinesConfigFromExportedZip = SaagieClientUtils.extractPipelineConfigAndPackageFromExportedPipeline(exportedArtifactsPathRoot) def variablesConfigFromExportedZip = SaagieClientUtils.extractVariableConfigAndPackageFromExportedVariable(exportedArtifactsPathRoot) + def appsConfigFromExportedZip = SaagieClientUtils.extractAppConfigAndPackageFromExportedApp(exportedArtifactsPathRoot) + def response = [ status : 'success', job : [], pipeline: [], - variable: [] + variable: [], + app : [], ] + def listJobs = null def processJobImportation = { newMappedJobData, job, id, versions = null, technologyName -> if (technologyName != null) { @@ -1988,6 +1996,84 @@ class SaagieClient { } + + + def listApps = null + def processAppToImport = { newMappedAppData, job, id, versions = null, technologyName -> + // check the for technology + if (technologyName != null) { + def foundTechnology = TechnologyService.instance.checkTechnologyIdExistInAppTechnologyList(technologyName); + if (!foundTechnology) { + throwAndLogError("Technology with name ${technologyName} is not available on the targeted server"); + } + newMappedAppData.job.technology = foundTechnology?.id + } + def appToImport = new App() + def appVersionToImport = new AppVersionDTO() + + appToImport = newMappedAppData.job + appVersionToImport = newMappedAppData.jobVersion + listApps = getAppListByProjectId() + boolean nameExist = false + def foundNameId = null + if (listApps) { + listApps.each { + if (it.name == newMappedAppData.job.name) { + nameExist = true + foundNameId = it.id + } + } + } + + def parsedNewlyCreatedApp = null + // change the app to Queue so we can remove the first + if (listApps && nameExist) { + appToImport.id = foundNameId + addAppVersion(appToImport, appVersionToImport) + + } else { + if (versions) { + versions.sort { a, b -> + return a.number?.toInteger() <=> b.number?.toInteger() + } + } + versions = versions as Queue + if (versions && versions.size() >= 1) { + def firstVersionInV1Format = versions.poll() + AppVersionDTO firstVersion = ImportAppService.convertFromMapToJsonVersion(firstVersionInV1Format) + appVersionToImport = firstVersion + } + def resultCreatedApp = createProjectApp(appToImport, appVersionToImport) + parsedNewlyCreatedApp = slurper.parseText(resultCreatedApp) + } + + response.app << [ + id : job.key, + name: newMappedAppData.job.name + ] + + if (versions && versions.size() >= 1) { + if (!parsedNewlyCreatedApp?.id && !foundNameId) { + throw new GradleException("Couldn't get id for the app after creation or update") + } + if (parsedNewlyCreatedApp?.id) { + appToImport.id = parsedNewlyCreatedApp?.id + }else { + appToImport.id = foundNameId + } + versions.each { + AppVersionDTO appVersionFromVersions = ImportAppService.convertFromMapToJsonVersion(it) + addAppVersion(appToImport, appVersionFromVersions) + } + + response.app.last() << [ + versions: versions.size() + 1 + ] + } + + } + + if (jobsConfigFromExportedZip?.jobs) { ImportJobService.importAndCreateJobs( jobsConfigFromExportedZip.jobs, @@ -2015,6 +2101,13 @@ class SaagieClient { ) } + if (appsConfigFromExportedZip?.apps) { + ImportAppService.importAndCreateApps( + appsConfigFromExportedZip.apps, + configuration, + processAppToImport + ) + } return response } catch (InvalidUserDataException invalidUserDataException) { throw invalidUserDataException @@ -2313,4 +2406,101 @@ class SaagieClient { def arrayApps = [] return arrayApps as ExportApp[] } + + private getAppListByProjectId() { + def listApps = null + tryCatchClosure({ + Request appsListRequest = saagieUtils.getAppListByProjectIdRequest() + client.newCall(appsListRequest).execute().withCloseable { responseAppList -> + handleErrors(responseAppList) + String responseBodyForAppList = responseAppList.body().string() + logger.debug("Apps with name and id : ") + logger.debug(responseBodyForAppList) + def parsedResultForAppList = slurper.parseText(responseBodyForAppList) + if (parsedResultForAppList.data?.labWebApps) { + listApps = parsedResultForAppList.data.labWebApps + } + return listApps + } + }, 'Unknown error in getAppListByProjectId', 'getAppListByProjectIdRequest Request') + } + + String addAppVersion(app, appVersion) { + // 2. add appVersion id there is a appVersion config + if (appVersion?.exists()) { + Request addAppVersionRequest = saagieUtils.updateAppVersion(app?.id, appVersion) + + client.newCall(addAppVersionRequest).execute().withCloseable { updateResponse -> + handleErrors(updateResponse) + String updateResponseBody = updateResponse.body().string() + def updatedAppVersion = slurper.parseText(updateResponseBody) + if (updatedAppVersion.data == null) { + def message = "Something went wrong when adding new app version: $updateResponseBody" + logger.error(message) + throw new GradleException(message) + } else { + String newAppVersion = updatedAppVersion.data.addJobVersion.number + logger.info('Added new version: {}', newAppVersion) + return newAppVersion + } + } + } else if (app?.id) { + Request listAppVersion = saagieUtils.getAppDetailRequest(app?.id) + client.newCall(listAppVersion).execute().withCloseable { listAppVersionsResponse -> + handleErrors(listAppVersionsResponse) + String listAppVersionResponseBody = listAppVersionsResponse.body().string() + def listAppVersionsData = slurper.parseText(listAppVersionResponseBody) + if (listAppVersionsData.data == null) { + def message = "Something went wrong when getting list app versions: $listAppVersionResponseBody" + logger.error(message) + throw new GradleException(message) + } else { + logger.info('getting list app versions: {}', listAppVersionsData.data.job.versions) + String currentNumber + listAppVersionsData.data.labWebApp.versions.each { + if (it.isCurrent) { + currentNumber = it.number + } + } + return currentNumber + } + } + } + } + + + String createProjectApp(App app, AppVersionDTO appVersion) { + logger.info('Starting deprecated createProjectApp task') + checkRequiredConfigForAppAndAppVersionAndProjectId(app, appVersion) + + + logger.debug('Using config [project={}, app={}, appVersion={}]', configuration.project, app, appVersion) + + Request projectCreateJobRequest = saagieUtils.getProjectCreateAppRequest(app, appVersion) + tryCatchClosure({ + client.newCall(projectCreateJobRequest).execute().withCloseable { response -> + handleErrors(response) + String responseBody = response.body().string() + def parsedResult = slurper.parseText(responseBody) + + if (parsedResult.data == null) { + def message = "Something went wrong when creating project app: $responseBody" + logger.error(message) + throw new GradleException(message) + } else { + Map createdApp = parsedResult.data.createJob + return JsonOutput.toJson(createdApp) + } + } + }, 'Unknown error in deprecated Task: createProjectApp', 'Function: createProjectApp') + } + + void checkRequiredConfigForAppAndAppVersionAndProjectId(App app, AppVersionDTO appVersion) { + checkRequiredConfig( + !configuration?.project?.id || + !app?.name || + !app?.technology || + !appVersion?.resources + ) + } } diff --git a/src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClientUtils.groovy b/src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClientUtils.groovy index ecf25c2a..133a714f 100644 --- a/src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClientUtils.groovy +++ b/src/main/groovy/io/saagie/plugin/dataops/clients/SaagieClientUtils.groovy @@ -7,6 +7,7 @@ class SaagieClientUtils { static final EXPORTED_JOB_PACKAGE_FOLDER_NAME = 'package' static final EXPORTED_PIPELINE_CONFIG_FILENAME = 'pipeline.json' static final EXPORTED_VARIABLE_CONFIG_FILENAME = 'variable.json' + static final EXPORTED_APP_CONFIG_FILENAME = 'app.json' /** * run through all files in the provided folder, and return @@ -45,6 +46,7 @@ class SaagieClientUtils { def jsonParser = new JsonSlurper() File jobConfigFile = new File("${jobFolderPath}/${EXPORTED_JOB_CONFIG_FILENAME}") extractedConfig.jobs[jobId].configOverride = jsonParser.parse(jobConfigFile) + if (extractedConfig.jobs[jobId].configOverride.versions) { extractedConfig.jobs[jobId].configOverride.versions.collect { def versionPackageName = new File("${jobFolderPath}/${EXPORTED_JOB_PACKAGE_FOLDER_NAME}/${it.number}").listFiles() @@ -126,4 +128,34 @@ class SaagieClientUtils { } return null } + + + def static extractAppConfigAndPackageFromExportedApp(File exportedAppFolder) { + Map extractedConfig = [ + apps: [:], + ] + + File appsFolder = new File("${exportedAppFolder.absolutePath}/App") + if (appsFolder.exists()) { + appsFolder.eachFile { appFolder -> + String appId = appFolder.name + String appFolderPath = appFolder.absolutePath + + extractedConfig.apps[appId] = [ + configOverride: null + ] + + def jsonParser = new JsonSlurper() + File appConfigFile = new File("${appFolderPath}/${EXPORTED_APP_CONFIG_FILENAME}"); + extractedConfig.apps[appId].configOverride = jsonParser.parse(appConfigFile) + } + + return extractedConfig + } + + return null + } + + + } diff --git a/src/main/groovy/io/saagie/plugin/dataops/models/App.groovy b/src/main/groovy/io/saagie/plugin/dataops/models/App.groovy index 7c969463..3e6b93c5 100644 --- a/src/main/groovy/io/saagie/plugin/dataops/models/App.groovy +++ b/src/main/groovy/io/saagie/plugin/dataops/models/App.groovy @@ -1,8 +1,14 @@ package io.saagie.plugin.dataops.models -class App { - List ids = [] - boolean include_all_versions = false - // TODO add feature include include_dockercredentials -// boolean include_dockercredentials = +class App extends Job { + int storageSizeInMB; + Boolean isStreaming; + + @Override + Map toMap() { + return [* : super.toMap(), + storageSizeInMB: storageSizeInMB, + isStreaming : isStreaming + ] + } } diff --git a/src/main/groovy/io/saagie/plugin/dataops/models/AppDTO.groovy b/src/main/groovy/io/saagie/plugin/dataops/models/AppDTO.groovy index 962a4ca2..d07c2584 100644 --- a/src/main/groovy/io/saagie/plugin/dataops/models/AppDTO.groovy +++ b/src/main/groovy/io/saagie/plugin/dataops/models/AppDTO.groovy @@ -2,9 +2,11 @@ package io.saagie.plugin.dataops.models class AppDTO extends JobDTO { int storageSizeInMB; + Boolean isStreaming; void setAppFromApiResult(appDetailResult) { this.storageSizeInMB = appDetailResult.storageSizeInMB; + this.isStreaming = appDetailResult.isStreaming; this.setJobFromApiResult(appDetailResult); } } diff --git a/src/main/groovy/io/saagie/plugin/dataops/models/AppMapper.groovy b/src/main/groovy/io/saagie/plugin/dataops/models/AppMapper.groovy index 589b05e0..c880228b 100644 --- a/src/main/groovy/io/saagie/plugin/dataops/models/AppMapper.groovy +++ b/src/main/groovy/io/saagie/plugin/dataops/models/AppMapper.groovy @@ -5,12 +5,12 @@ class AppMapper { AppVersionDTO jobVersion = new AppVersionDTO() - Object app(Closure closure) { - app.with(closure) + Object job(Closure closure) { + job.with(closure) } - Object appVersion(Closure closure) { - appVersion.with(closure) + Object jobVersion(Closure closure) { + jobVersion.with(closure) } def static Map mapAppAndAppVersionWithoutMail(App app, AppVersionDTO appVersion, String projectId) { diff --git a/src/main/groovy/io/saagie/plugin/dataops/models/AppVersionDTO.groovy b/src/main/groovy/io/saagie/plugin/dataops/models/AppVersionDTO.groovy index 415c8fe7..ccd11fbc 100644 --- a/src/main/groovy/io/saagie/plugin/dataops/models/AppVersionDTO.groovy +++ b/src/main/groovy/io/saagie/plugin/dataops/models/AppVersionDTO.groovy @@ -1,6 +1,6 @@ package io.saagie.plugin.dataops.models -class AppVersionDTO implements IExists { +class AppVersionDTO implements IMapable,IExists { List storagePaths = [] String releaseNote int number @@ -8,6 +8,32 @@ class AppVersionDTO implements IExists { DockerInfos dockerInfo = new DockerInfos() Resources resources = new Resources() + Object dockerInfo(Closure closure) { + dockerInfo.with(closure) + } + + Object resources(Closure closure) { + resources.with(closure) + } + + Object exposedPorts(Closure closure) { + exposedPorts.with(closure) + } + + + @Override + Map toMap() { + if (exists()) { + return [ + releaseNote : releaseNote, + exposedPorts : exposedPorts ? exposedPorts as List : [], + storagePaths : storagePaths ? storagePaths as List : [], + dockerInfo : dockerInfo.toMap(), + resources : resources.toMap(), + ] + } + return null + } @Override boolean exists() { return dockerInfo || diff --git a/src/main/groovy/io/saagie/plugin/dataops/tasks/service/TechnologyService.groovy b/src/main/groovy/io/saagie/plugin/dataops/tasks/service/TechnologyService.groovy index 481525bf..ac07b8db 100644 --- a/src/main/groovy/io/saagie/plugin/dataops/tasks/service/TechnologyService.groovy +++ b/src/main/groovy/io/saagie/plugin/dataops/tasks/service/TechnologyService.groovy @@ -19,17 +19,21 @@ class TechnologyService { JsonSlurper slurper static final Logger logger = Logging.getLogger(TechnologyService.class) def isInitialised = false + def appTechnologyList + def appTechnologiesRequest def init( client, technologiesRequest, technologiesVersionsRequest, - slurper + slurper, + allTechnologiesForAppRequest ) { this.client = client this.slurper = slurper this.technologiesRequest = technologiesRequest this.technologiesVersionsRequest = technologiesVersionsRequest + this.appTechnologiesRequest = allTechnologiesForAppRequest this.isInitialised = true } @@ -230,4 +234,39 @@ class TechnologyService { def convertToNumber(String technologyLabel) { return technologyLabel.findAll(/\d+/)*.toInteger().join().toInteger() } + + def getAppTechnologies() { + checkReady() + Request appTechnologiesCall = appTechnologiesRequest() + + if (!appTechnologyList) { + + client.newCall(appTechnologiesCall).execute().withCloseable { response -> + handleErrors(response) + String responseBody = response.body().string() + logger.debug("getAppTechnologies response $responseBody") + def parsedTechnologiesData = slurper.parseText(responseBody) + if (!parsedTechnologiesData.data || !parsedTechnologiesData.data.repositories) { + throwAndLogError("Something went wrong when getting app technologies") + } + def repo = parsedTechnologiesData.data.repositories + appTechnologyList = this.getTechnologiesFromRepositories(repo) + } + } + return appTechnologyList + } + + def checkTechnologyIdExistInAppTechnologyList(String technologyId) { + def tech = this.getAppTechnologies().find{it.label?.equals(technologyId)} + return tech + } + + def getTechnologiesFromRepositories(repositories){ + def technologyList = [] + repositories.forEach{ repositorie -> + def list = repositorie.technologies as List + technologyList=(technologyList< + def versions = null + def appId = app.key + Map appConfig = app.value.configOverride + + AppMapper newMappedAppData = new AppMapper() + + newMappedAppData.job { + name = appConfig.app.name + technology = appConfig.app.technology + isScheduled = appConfig.app.isScheduled + category = appConfig.app.category + description = appConfig.app.description + storageSizeInMB = appConfig.app.storageSizeInMB + isStreaming = appConfig.app.isStreaming + } + + if (appConfig.app.alerting?.emails) { + newMappedAppData.job.alerting { + emails = appConfig.app.alerting?.emails + statusList = appConfig.app.alerting?.statusList + logins = appConfig.app.alerting?.logins + } + } + + newMappedAppData.jobVersion { + releaseNote = appConfig.appVersion.releaseNote + + dockerInfo { + image = appConfig.appVersion.dockerInfo?.image + dockerCredentialsId = appConfig.appVersion.dockerInfo?.dockerCredentialsId + } + + resources { + memory = appConfig.appVersion.resources.memory + cpu = appConfig.appVersion.resources.cpu + disk = appConfig.appVersion.resources.disk + } + + exposedPorts = appConfig.appVersion.exposedPorts + + } + + if (appConfig.versions && appConfig.versions.size() > 0) { + versions = appConfig.versions + } + + + mapClosure(newMappedAppData, app, app.key, versions, appConfig.app.technologyName) + } + + } + + static convertFromMapToJsonVersion(appVersionMap) { + AppVersionDTO appVersion = [] + appVersion.with { + releaseNote = appVersionMap.releaseNote + + dockerInfo { + image = appVersionMap.dockerInfo?.image + dockerCredentialsId = appVersionMap.dockerInfo?.dockerCredentialsId + } + resources { + memory = appVersionMap.resources.memory + cpu = appVersionMap.resources.cpu + disk = appVersionMap.resources.disk + } + + exposedPorts = appVersionMap.exposedPorts + + } + + return appVersion + } + +} diff --git a/src/main/groovy/io/saagie/plugin/dataops/utils/SaagieUtils.groovy b/src/main/groovy/io/saagie/plugin/dataops/utils/SaagieUtils.groovy index 0179cc38..6e80354e 100644 --- a/src/main/groovy/io/saagie/plugin/dataops/utils/SaagieUtils.groovy +++ b/src/main/groovy/io/saagie/plugin/dataops/utils/SaagieUtils.groovy @@ -366,18 +366,19 @@ class SaagieUtils { def gqVariables = jsonGenerator.toJson([id: appId]) def getAppDetailQuery = gq(''' - query labWebApp($id: UUID!) { labWebApp(id: $id) { id name description creationDate isDeletable storageSizeInMB instances(limit: 1, checkInPipelineInstance: false) { id status statusDetails startTime endTime version { number } } versions { number creator creationDate number isCurrent releaseNote resources{cpu memory disk} dockerInfo { image dockerCredentialsId } exposedPorts { name port isAuthenticationRequired isRewriteUrl basePathVariableName } storagePaths } alerting { emails statusList loginEmails { login email } } technology { id label } }} + query labWebApp($id: UUID!) { labWebApp(id: $id) { id name description creationDate isDeletable category isScheduled isStreaming storageSizeInMB instances(limit: 1, checkInPipelineInstance: false) { id status statusDetails startTime endTime version { number } } versions { number creator creationDate number isCurrent releaseNote resources{cpu memory disk} dockerInfo { image dockerCredentialsId } exposedPorts { name port isAuthenticationRequired isRewriteUrl basePathVariableName } storagePaths } alerting { emails statusList loginEmails { login email } } technology { id label } }} ''', gqVariables) return buildRequestFromQuery(getAppDetailQuery) } - Request getAppListByProjectIdRequest(String projectId) { - logger.debug('Generating app list for project with id', projectId) + Request getAppListByProjectIdRequest() { + Project project = configuration.project + logger.debug('Generating app list for project with id', project.id) def jsonGenerator = new JsonGenerator.Options() .build() - def gqVariables = jsonGenerator.toJson([projectId: projectId]) + def gqVariables = jsonGenerator.toJson([projectId: project.id]) def getAppsListQuery = gq(''' query labWebAppsQuery($projectId: UUID!) { labWebApps(projectId: $projectId) { id name }} @@ -385,13 +386,13 @@ class SaagieUtils { return buildRequestFromQuery(getAppsListQuery) } - Request getAppTechnologiesList(String appId) { - logger.debug('Generating technologies for application with id', appId) + Request getAppTechnologiesList() { + logger.debug('Generating technologies for application') def jsonGenerator = new JsonGenerator.Options() .build() - def gqVariables = jsonGenerator.toJson([id: appId]) + def gqVariables = jsonGenerator.toJson([:]) def getAppTechnologiesListQuery = gq(''' query repositoriesQuery { repositories { id name technologies { ... on AppTechnology { id label description icon backgroundColor available customFlags } } }} @@ -399,15 +400,15 @@ class SaagieUtils { return buildRequestFromQuery(getAppTechnologiesListQuery, true, true) } - Request updateAppVersion(String appId, AppVersionDTO appVersionDTO ) { + Request updateAppVersion(String appId, AppVersionDTO appVersionDTO) { logger.debug('Updating application version for application with id', appId) def jsonGenerator = new JsonGenerator.Options() .build() def gqVariables = jsonGenerator.toJson([ - appId: appId, - appVersion: appVersionDTO + appId : appId, + appVersion: appVersionDTO.toMap() ]) def getUpdateAppVersionQuery = gq(''' @@ -550,7 +551,7 @@ class SaagieUtils { Request getProjectCreateJobRequestWithGraphQL(Job job, JobVersion jobVersion) { Map mapedJobAndJobVersion = JobMapper.mapJobAndJobVersionWithoutMail(job, jobVersion, configuration.project.id) - logger.debug('Generating getProjectCreateJobRequest [job={}, jobVersion={}]', job, jobVersion) + logger.debug('Generating getProjectCreateJobRequest [job={}, jobVersion={}]', job, jobVersion) File file = new File(jobVersion.packageInfo.name) logger.debug('Using [file={}] for upload', file.absolutePath) diff --git a/src/main/groovy/io/saagie/plugin/dataops/utils/directory/FolderGenerator.groovy b/src/main/groovy/io/saagie/plugin/dataops/utils/directory/FolderGenerator.groovy index 049b5584..1d2ab8ac 100644 --- a/src/main/groovy/io/saagie/plugin/dataops/utils/directory/FolderGenerator.groovy +++ b/src/main/groovy/io/saagie/plugin/dataops/utils/directory/FolderGenerator.groovy @@ -137,12 +137,14 @@ class FolderGenerator { def createFolderForApp = folder.mkdirs() if (createFolderForApp) { Map appDetailObject = [ - name : exportApp.appDTO.name, - category : exportApp.appDTO.category, - technology : exportApp.appDTO.technology, - technologyName: exportApp.appDTO.technologyName, - isScheduled : exportApp.appDTO.isScheduled, + name : exportApp.appDTO.name, + category : exportApp.appDTO.category, + technology : exportApp.appDTO.technology, + technologyName : exportApp.appDTO.technologyName, + isScheduled : exportApp.appDTO.isScheduled, storageSizeInMB: exportApp.appDTO.storageSizeInMB, + isStreaming : exportApp.appDTO.isStreaming, + description : exportApp.appDTO.description, ] if ( @@ -154,11 +156,7 @@ class FolderGenerator { ]] } - if (exportApp.appDTO?.description) { - appDetailObject << [*: [ - description: exportApp.appDTO?.description - ]] - } + def appVersionDetailJsonObject = generateAppVersion(exportApp.appVersionDTO) Map appJsonObject = [ app : appDetailObject, @@ -203,7 +201,7 @@ class FolderGenerator { if (appVersionDTO?.resources?.cpu && appVersionDTO?.resources?.disk && - appVersionDTO?.resources?.memory ) { + appVersionDTO?.resources?.memory) { appVersionDetailJsonObject << [*: [ resources: appVersionDTO.resources, ]] diff --git a/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportTaskTest.groovy b/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportTaskTest.groovy index 37b0f35f..b6b19843 100644 --- a/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportTaskTest.groovy +++ b/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportTaskTest.groovy @@ -23,6 +23,13 @@ class ArtifactsImportTaskTest extends DataOpsGradleTaskSpecification { String exportJobWithoutPipelineZipFilename = './exportedJobWithoutPipeline.zip' @Shared String exportJobJustJobVersionWithoutPipelineZipFilename = './exportJobJustJobVersionWithoutPipelineZipFilename.zip' + @Shared + String exportAppZipFilename = './exportedApp.zip' + @Shared + String exportAppWithAppVersionZipFilename = './exportedAppWithAppVersion.zip' + String exportAppWithAppWithTechnologyNameDoesntExistVersionZipFilename = './exportedAppWithAppVersionNameDoesntExist.zip' + + def "the task should fail if required params are not provided"() { given: @@ -122,7 +129,7 @@ class ArtifactsImportTaskTest extends DataOpsGradleTaskSpecification { then: notThrown(Exception) - result.output.contains('{status=success, job=[{id=id-1, name=Test Job3 imported from file}], pipeline=[{id=id-1, name=test pipeline 23}], variable=[]}') + result.output.contains('{status=success, job=[{id=id-1, name=Test Job3 imported from file}], pipeline=[{id=id-1, name=test pipeline 23}], variable=[], app=[]}') } def "the task should create a new pipeline and add new version to another pipeline without job based on the exported config"() { @@ -159,9 +166,49 @@ class ArtifactsImportTaskTest extends DataOpsGradleTaskSpecification { then: notThrown(Exception) - result.output.contains('{status=success, job=[], pipeline=[{id=id-2, name=test pipeline exist}, {id=id-1, name=test pipeline 23}], variable=[]}') + result.output.contains('{status=success, job=[], pipeline=[{id=id-2, name=test pipeline exist}, {id=id-1, name=test pipeline 23}], variable=[], app=[]}') } + + + + + def "the task should add appVersion based on the build configuration if name exist"() { + given: + URL resource = classLoader.getResource(exportAppWithAppVersionZipFilename) + File exportedConfig = new File(resource.getFile()) + enqueueRequest('{"data":{"repositories":[{"id":"id-tech-1","name":"Saagie","technologies":[{"id":"id-tech-2","label":"tech-1","description":"description-1","icon":"icon-1","backgroundColor":null,"available":true,"customFlags":["flag-1"]},{"id":"id-tech-3","label":"tech-2","description":"des-3","icon":"icon-3","backgroundColor":"#E87A35","available":true,"customFlags":[]},{"id":"id-tech-4","label":"tech-4","description":"desc-4","icon":"icon-4","backgroundColor":"#728E9B","available":true,"customFlags":[]}]}]}}') + enqueueRequest('{"data":{"labWebApps":[{"id":"id-app-1","name":"app-1"},{"id":"id-app-2","name":"testImport7"},{"id":"id-app-3","name":"testImport6"},{"id":"id-app-4","name":"testImport5"},{"id":"id-app-5","name":"testImport4"},{"id":"id-app-6","name":"testImport3"},{"id":"id-app-7","name":"testImport2"},{"id":"id-app-8","name":"testImport"}]}}') + enqueueRequest('{"data":{"addJobVersion":{"number":2}}}') + + buildFile << """ + saagie { + server { + url = '${mockServerUrl}' + login = 'login' + password = 'password' + environment = 1 + } + + project { + id = 'project-id' + } + + importArtifacts { + import_file = '${exportedConfig.absolutePath}' + } + } + """ + + when: + BuildResult result = gradle(taskName) + + then: + notThrown(Exception) + result.output.contains('{status=success, job=[], pipeline=[], variable=[], app=[{id=id-1, name=app-1}]}') + } + + def "the task should create a new pipeline and new job based on the exported config if name doesn't exist"() { given: URL resource = classLoader.getResource(exportJobZipFilename) @@ -196,9 +243,46 @@ class ArtifactsImportTaskTest extends DataOpsGradleTaskSpecification { then: notThrown(Exception) - result.output.contains('{status=success, job=[{id=id-1, name=Test Job3 imported from file}], pipeline=[{id=id-1, name=test pipeline 23}], variable=[]}') + result.output.contains('{status=success, job=[{id=id-1, name=Test Job3 imported from file}], pipeline=[{id=id-1, name=test pipeline 23}], variable=[], app=[]}') } + + def "the task should add jobVersion based on the build configuration if name exist with overwrite"() { + + given: + URL resource = classLoader.getResource(exportJobJustJobVersionWithoutPipelineZipFilename) + File exportedConfig = new File(resource.getFile()) + enqueueRequest('{"data":{"jobs":[{"id":"id-1","name":"Job from import asdas"},{"id":"id-2","name":"test added job"},{"id":"id-3","name":"name job 3"}]}}') + enqueueRequest('{"data":{"addJobVersion":{"number":"jobNumber"}}}') + + buildFile << """ + saagie { + server { + url = '${mockServerUrl}' + login = 'login' + password = 'password' + environment = 1 + } + + project { + id = 'project-id' + } + + importArtifacts { + import_file = '${exportedConfig.absolutePath}' + } + } + """ + + when: + BuildResult result = gradle(taskName) + + then: + notThrown(Exception) + result.output.contains('{status=success, job=[{id=id-1, name=test added job}], pipeline=[], variable=[], app=[]}') + } + + def "the task should create new job and add new version to another job without pipeline based on the exported config"() { given: URL resource = classLoader.getResource(exportJobWithoutPipelineZipFilename) @@ -232,16 +316,18 @@ class ArtifactsImportTaskTest extends DataOpsGradleTaskSpecification { then: notThrown(Exception) - result.output.contains('{status=success, job=[{id=id-2, name=name exist}, {id=id-1, name=test added job}], pipeline=[], variable=[]}') + result.output.contains('{status=success, job=[{id=id-2, name=name exist}, {id=id-1, name=test added job}], pipeline=[], variable=[], app=[]}') } - def "the task should add jobVersion based on the build configuration if name exist with overwrite"() { + def "the task should create new app and add new version to another app based on the exported config"() { given: - URL resource = classLoader.getResource(exportJobJustJobVersionWithoutPipelineZipFilename) + URL resource = classLoader.getResource(exportAppZipFilename) File exportedConfig = new File(resource.getFile()) - enqueueRequest('{"data":{"jobs":[{"id":"id-1","name":"Job from import asdas"},{"id":"id-2","name":"test added job"},{"id":"id-3","name":"name job 3"}]}}') - enqueueRequest('{"data":{"addJobVersion":{"number":"jobNumber"}}}') + enqueueRequest('{"data":{"repositories":[{"id":"id-tech-1","name":"Saagie","technologies":[{"id":"id-tech-2","label":"tech-2","description":"description-1","icon":"icon-1","backgroundColor":null,"available":true,"customFlags":["flag-1"]},{"id":"id-tech-3","label":"tech-3","description":"des-3","icon":"icon-3","backgroundColor":"#E87A35","available":true,"customFlags":[]},{"id":"id-tech-4","label":"tech-4","description":"desc-4","icon":"icon-4","backgroundColor":"#728E9B","available":true,"customFlags":[]}]}]}}') + enqueueRequest('{"data":{"labWebApps":[{"id":"id-app-1","name":"testImport8"},{"id":"id-app-2","name":"testImport7"},{"id":"id-app-3","name":"testImport6"},{"id":"id-app-4","name":"testImport5"},{"id":"id-app-5","name":"testImport4"},{"id":"id-app-6","name":"testImport3"},{"id":"id-app-7","name":"testImport2"},{"id":"id-app-8","name":"testImport"}]}}') + enqueueRequest('{"data":{"createJob":{"id":"id-app-10","name":"testImport9"}}}') + enqueueRequest('{"data":{"addJobVersion":{"number":2}}}') buildFile << """ saagie { @@ -267,8 +353,41 @@ class ArtifactsImportTaskTest extends DataOpsGradleTaskSpecification { then: notThrown(Exception) - result.output.contains('{status=success, job=[{id=id-1, name=test added job}], pipeline=[], variable=[]}') + result.output.contains('{status=success, job=[], pipeline=[], variable=[], app=[{id=id-1, name=testImport9}]}') } + def "the task should fail if the app's technology does not exists"() { + + given: + URL resource = classLoader.getResource(exportAppWithAppWithTechnologyNameDoesntExistVersionZipFilename) + File exportedConfig = new File(resource.getFile()) + enqueueRequest('{"data":{"repositories":[{"id":"id-tech-1","name":"Saagie","technologies":[{"id":"id-tech-2","label":"tech-1","description":"description-1","icon":"icon-1","backgroundColor":null,"available":true,"customFlags":["flag-1"]},{"id":"id-tech-3","label":"tech-3","description":"des-3","icon":"icon-3","backgroundColor":"#E87A35","available":true,"customFlags":[]},{"id":"id-tech-4","label":"tech-4","description":"desc-4","icon":"icon-4","backgroundColor":"#728E9B","available":true,"customFlags":[]}]}]}}') + + buildFile << """ + saagie { + server { + url = '${mockServerUrl}' + login = 'login' + password = 'password' + environment = 1 + } + + project { + id = 'project-id' + } + importArtifacts { + import_file = '${exportedConfig.absolutePath}' + } + } + """ + when: + BuildResult result = gradle(taskName) + + then: + UnexpectedBuildFailure e = thrown() + result == null + e.message.contains("Technology with name Mlflow Server name doesnt exist is not available on the targeted server") + e.getBuildResult().task(":${taskName}").outcome == FAILED + } } diff --git a/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportV1Test.groovy b/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportV1Test.groovy index 15143cbb..c673f984 100644 --- a/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportV1Test.groovy +++ b/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportV1Test.groovy @@ -20,7 +20,7 @@ class ArtifactsImportV1Test extends DataOpsGradleTaskSpecification { given: URL resource = classLoader.getResource(exportedPipelineWithoutJobForV1) File exportedConfig = new File(resource.getFile()) - enqueueRequest('{"data":{"jobs":[{"id":"job-1","name":"job name not contained in the pipeline version"}, {"id":"job-2","name":"job name name contained in the pipeline version"}]}}') + enqueueRequest('{"data":{"jobs":[{"id":"job-1","name":"job name not contained in the pipeline version"}, {"id":"job-2","name":"job name name contained in the pipeline version"}]}}') buildFile << """ saagie { @@ -42,7 +42,7 @@ class ArtifactsImportV1Test extends DataOpsGradleTaskSpecification { """ when: - BuildResult result = gradle(taskName, "-d") + BuildResult result = gradle(taskName) then: UnexpectedBuildFailure e = thrown() diff --git a/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportV2EnvVarTaskTest.groovy b/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportV2EnvVarTaskTest.groovy index c968d68b..9b8b7b6e 100644 --- a/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportV2EnvVarTaskTest.groovy +++ b/src/test/groovy/io/saagie/plugin/tasks/artifacts/ArtifactsImportV2EnvVarTaskTest.groovy @@ -43,6 +43,6 @@ class ArtifactsImportV2EnvVarTaskTest extends DataOpsGradleTaskSpecification { then: notThrown(Exception) - result.output.contains('{status=success, job=[], pipeline=[], variable=[{id=variable-1, name=GLOBAL_AMINE}, {id=variable-2, name=testUX2}]}') + result.output.contains('{status=success, job=[], pipeline=[], variable=[{id=variable-1, name=GLOBAL_AMINE}, {id=variable-2, name=testUX2}], app=[]}') } } diff --git a/src/test/resources/exportedApp.zip b/src/test/resources/exportedApp.zip new file mode 100644 index 00000000..642c09c8 Binary files /dev/null and b/src/test/resources/exportedApp.zip differ diff --git a/src/test/resources/exportedAppWithAppVersion.zip b/src/test/resources/exportedAppWithAppVersion.zip new file mode 100644 index 00000000..30d7498e Binary files /dev/null and b/src/test/resources/exportedAppWithAppVersion.zip differ diff --git a/src/test/resources/exportedAppWithAppVersionNameDoesntExist.zip b/src/test/resources/exportedAppWithAppVersionNameDoesntExist.zip new file mode 100644 index 00000000..6ce9f76c Binary files /dev/null and b/src/test/resources/exportedAppWithAppVersionNameDoesntExist.zip differ