Add repositories info so the generation of docs TIP and overall TIR have the info they need to render the doc.

CHANGELOG.md

@@ -12,9 +12,8 @@
 - Added IT (Docker tests)
 - Added performance tests
 - logback.xml can be overridden from command line
-- removed unused params from payloads.
-- removed param jekins to jenkins
-- Removed byte[] use
+- removed unused params from payloads, jekins -> jenkins, computes values gitUrl and doInstall for each repository.
+- Removed byte[] usage
 
 ## [4.0] - 2021-18-11
 =======

src/main/groovy/org/ods/doc/gen/external/modules/git/BitBucketRepository.groovy

@@ -28,4 +28,11 @@ interface BitBucketRepository {
                                @Param("repo") String repo,
                                @Param("branch") String branch)
 
+    @Headers("Accept: application/octet-stream")
+    @RequestLine("GET /rest/api/latest/projects/{project}/repos/{repo}/archive?at={branch}&format=zip&path={filePath}")
+    Response getRepoFileInZipArchive(@Param("project") String project,
+                                     @Param("repo") String repo,
+                                     @Param("branch") String branch,
+                                     @Param("filePath") String filePath)
+
 }

src/main/groovy/org/ods/doc/gen/external/modules/git/BitbucketService.groovy

@@ -20,8 +20,6 @@ import java.nio.file.StandardCopyOption
 @Service
 class BitbucketService {
 
-    private static final String MAIN_BRANCH = "master"
-
     private ZipFacade zipFacade
     private final BitBucketClientConfig bitBucketClientConfig
 
@@ -44,8 +42,8 @@ class BitbucketService {
         return new JsonSlurperClassic().parseText(response)
     }
 
-    String buildReleaseManagerUrl(String projectId, String releaseManagerRepo) {
-        URI uri = new URI([getBitbucketURLForDocs(), projectId, releaseManagerRepo].join("/"))
+    String buildRepositoryUrl(String projectId, String repositoryName) {
+        URI uri = new URI([getBitbucketURLForDocs(), projectId, repositoryName].join("/"))
         return "${uri.normalize().toString()}.git"
     }
 
@@ -54,36 +52,70 @@ class BitbucketService {
     }
 
     void downloadRepo(String project, String repo, String branch, String tmpFolder) {
+        downloadRepoWithFallBack(project, repo, branch, null, tmpFolder)
+    }
+
+    void downloadRepoWithFallBack(String project, String repo, String branch, String defaultBranch,
+                                            String tmpFolder) {
         log.info("downloadRepo: project:${project}, repo:${repo} and branch:${branch}")
-        Path zipArchive = Files.createTempFile("archive-", ".zip")
+        Path zipArchive = Files.createTempFile("archive-${repo}", ".zip")
         try {
-            downloadRepoWithFallBack(project, repo, branch, zipArchive)
+            try {
+                bitBucketClientConfig
+                        .getClient()
+                        .getRepoZipArchive(project, repo, branch)
+                        .withCloseable { Response response ->
+                            streamResult(response, zipArchive)
+                        }
+            } catch (Exception firstTryException) {
+                if (defaultBranch) {
+                    log.warn("Branch [${branch}] doesn't exist, using branch: [${defaultBranch}]")
+                    bitBucketClientConfig
+                            .getClient()
+                            .getRepoZipArchive(project, repo, defaultBranch)
+                            .withCloseable { Response response ->
+                                streamResult(response, zipArchive)
+                            }
+                } else {
+                    throw firstTryException
+                }
+            }
             zipFacade.extractZipArchive(zipArchive, Paths.get(tmpFolder))
         } catch (FeignException callException) {
             checkError(repo, branch, callException)
         } finally {
             Files.delete(zipArchive)
         }
+
     }
 
-    protected void downloadRepoWithFallBack(String project, String repo, String branch, Path zipArchive) {
+    void downloadRepoMetadata(String project, String repo, String branch, String defaultBranch, String tmpFolder) {
+        log.info("downloadRepo: project:${project}, repo:${repo} and branch:${branch}")
+        Path zipArchive = Files.createTempFile("archive-${repo}", ".zip")
         try {
-            bitBucketClientConfig
-                    .getClient()
-                    .getRepoZipArchive(project, repo, branch)
-                    .withCloseable { Response response ->
-                        streamResult(response, zipArchive)
-                    }
-        } catch (Exception callException) {
-            log.warn("Branch [${branch}] doesn't exist, using branch: [${MAIN_BRANCH}]")
-            bitBucketClientConfig
-                    .getClient()
-                    .getRepoZipArchive(project, repo, MAIN_BRANCH)
-                    .withCloseable { Response response ->
-                        streamResult(response, zipArchive)
-                    }
-        }
+            try {
+                bitBucketClientConfig
+                        .getClient()
+                        .getRepoFileInZipArchive(project, repo, branch, "metadata.yml")
+                        .withCloseable { Response response ->
+                            streamResult(response, zipArchive)
+                        }
+            } catch (e) {
+                log.warn("Branch [${branch}] doesn't exist, using branch: [${defaultBranch}]")
+                bitBucketClientConfig
+                        .getClient()
+                        .getRepoFileInZipArchive(project, repo, defaultBranch, "metadata.yml")
+                        .withCloseable { Response response ->
+                            streamResult(response, zipArchive)
+                        }
+            }
 
+            zipFacade.extractZipArchive(zipArchive, Paths.get(tmpFolder))
+        } catch (FeignException callException) {
+            checkError(repo, branch, callException)
+        } finally {
+            Files.delete(zipArchive)
+        }
     }
 
     protected void streamResult(Response response, Path zipArchive){

src/main/groovy/org/ods/doc/gen/leva/doc/services/LeVADocumentService.groovy

@@ -2,6 +2,7 @@ package org.ods.doc.gen.leva.doc.services
 
 import groovy.util.logging.Slf4j
 import groovy.xml.XmlUtil
+import org.ods.doc.gen.external.modules.git.BitbucketService
 import org.ods.doc.gen.external.modules.git.BitbucketTraceabilityUseCase
 import org.ods.doc.gen.external.modules.jira.CustomIssueFields
 import org.ods.doc.gen.external.modules.jira.IssueTypes
@@ -63,6 +64,7 @@ class LeVADocumentService {
     private final LeVADocumentChaptersFileService levaFiles
     private final BitbucketTraceabilityUseCase bbt
     private final NexusService nexus
+    private final BitbucketService bitbucketService
 
     @Inject
     Clock clock
@@ -74,14 +76,16 @@ class LeVADocumentService {
                         JiraUseCase jiraUseCase,
                         JUnitReportsService junit,
                         LeVADocumentChaptersFileService levaFiles,
-                        BitbucketTraceabilityUseCase bbt) {
+                        BitbucketTraceabilityUseCase bbt,
+                        BitbucketService bitbucketService) {
         this.project = project
         this.docGenUseCase = docGenUseCase
         this.nexus = nexus
         this.jiraUseCase = jiraUseCase
         this.junit = junit
         this.levaFiles = levaFiles
         this.bbt = bbt
+        this.bitbucketService = bitbucketService
     }
 
     @SuppressWarnings('CyclomaticComplexity')
@@ -364,14 +368,14 @@ class LeVADocumentService {
         def keysInDoc = this.computeKeysInDocForIPV(installationTestIssues)
         def docHistory = this.getAndStoreDocumentHistory(documentType, keysInDoc, projectData)
 
-        def installedRepos = projectData.repositories.collect {
-            it << [ doInstall: !Constants.COMPONENT_TYPE_IS_NOT_INSTALLED.contains(it.type?.toLowerCase())]
-        }
+        updateProjectRepositoriesDoInstall(projectData)
 
         def data_ = [
                 metadata: this.getDocumentMetadata(projectData, Constants.DOCUMENT_TYPE_NAMES[documentType]),
                 data    : [
-                        repositories   : installedRepos.collect { [id: it.id, type: it.type, doInstall: it.doInstall, data: [git: [url: it.data.git == null ? null : it.data.git.url]]] },
+                        repositories   : projectData.repositories.collect {
+                            [id: it.id, type: it.type, doInstall: it.doInstall,
+                             data: [git: [url: it.metadata.gitUrl]]] },
                         sections       : sections,
                         tests          : SortUtil.sortIssuesByKey(installationTestIssues.collect { testIssue ->
                             [
@@ -522,6 +526,8 @@ class LeVADocumentService {
         def keysInDoc = this.computeKeysInDocForTIP(projectData.getComponents())
         def docHistory = this.getAndStoreDocumentHistory(documentType, keysInDoc, projectData)
 
+        updateRepositoriesData(data)
+
         def data_ = [
                 metadata: this.getDocumentMetadata(projectData, Constants.DOCUMENT_TYPE_NAMES[documentType]),
                 data    : [
@@ -753,14 +759,14 @@ class LeVADocumentService {
         def keysInDoc =  this.computeKeysInDocForIVR(installationTestIssues)
         def docHistory = this.getAndStoreDocumentHistory(documentType, keysInDoc, projectData)
 
-        def installedRepos = projectData.repositories.collect {
-            it << [ doInstall: !Constants.COMPONENT_TYPE_IS_NOT_INSTALLED.contains(it.type?.toLowerCase())]
-        }
+        updateProjectRepositoriesDoInstall(projectData)
 
         def data_ = [
                 metadata: this.getDocumentMetadata(projectData, Constants.DOCUMENT_TYPE_NAMES[documentType]),
                 data    : [
-                        repositories   : installedRepos.collect { [id: it.id, type: it.type, doInstall: it.doInstall, data: [git: [url: it.data.git == null ? null : it.data.git.url]]] },
+                        repositories   : projectData.repositories.collect {
+                            [id: it.id, type: it.type, doInstall: it.doInstall,
+                             data: [git: [url: it.metadata.gitUrl]]] },
                         sections          : sections,
                         tests             : buildTestResultsIVR(installationTestIssues),
                         numAdditionalTests: getNumAdditionalTest(installationTestData, installationTestIssues),
@@ -945,6 +951,8 @@ class LeVADocumentService {
 
         def watermarkText = this.getWatermarkText(projectData)
 
+        updateRepositoriesData(data)
+
         def visitor = { data_ ->
             // Prepend a section for the Jenkins build log
             data_.sections.add(0, [
@@ -1243,7 +1251,7 @@ class LeVADocumentService {
         projectData.repositories.collect {
             [
                 id: it.id,
-                description: it.metadata?.description,
+                description: it.metadata.description,
                 tests: componentTestMapping[it.id]? componentTestMapping[it.id].join(", "): "None defined"
             ]
         }
@@ -1639,4 +1647,21 @@ class LeVADocumentService {
         junit.getNumberOfTestCases(testData.testResults) - testIssues.count { !it.isUnexecuted }
     }
 
+    private void updateProjectRepositoriesDoInstall(ProjectData projectData) {
+        projectData.repositories.forEach({
+            it.doInstall = !Constants.COMPONENT_TYPE_IS_NOT_INSTALLED.contains(it.type?.toLowerCase())
+        })
+    }
+
+    private updateRepositoriesData(Map data) {
+        String projectId = data.projectId as String
+
+        data.repositories.each { Map repo ->
+            String repoName = repo.id as String
+            repoName = "${projectId}-${repoName}"
+            repo["data"]["git"]["url"] = bitbucketService.buildRepositoryUrl(projectId, repoName)
+
+            repo["doInstall"] = !Constants.COMPONENT_TYPE_IS_NOT_INSTALLED.contains(repo.type?.toLowerCase())
+        }
+    }
 }

src/main/groovy/org/ods/doc/gen/project/data/ProjectData.groovy

@@ -18,6 +18,9 @@ import org.yaml.snakeyaml.Yaml
 
 import java.nio.file.Paths
 
+import static groovy.json.JsonOutput.prettyPrint
+import static groovy.json.JsonOutput.toJson
+
 @SuppressWarnings(['LineLength',
         'AbcMetric',
         'IfStatementBraces',
@@ -32,8 +35,10 @@ import java.nio.file.Paths
 @Service
 class ProjectData {
 
+    static final String DEFAULT_TEMPLATE_VERSION = '1.2'
     static final String BUILD_PARAM_VERSION_DEFAULT = 'WIP'
     static final String METADATA_FILE_NAME = 'metadata.yml'
+    private static final String DEFAULT_BRANCH_RELEASE_MANAGER = "master"
 
     protected Map config
     protected Boolean isVersioningEnabled = false
@@ -69,18 +74,19 @@ class ProjectData {
         this.data.jira = [project: [ : ]]
         this.data.repo = data.repo
         this.data.git = data.git
-        this.data.git.url = bitbucketService.buildReleaseManagerUrl(
+        this.data.git.url = bitbucketService.buildRepositoryUrl(
                 data.projectId as String,
                 data.git.releaseManagerRepo as String
         )
         return this
     }
 
     ProjectData load() {
-        bitbucketService.downloadRepo(
+        bitbucketService.downloadRepoWithFallBack(
                 data.projectId as String,
                 data.git.releaseManagerRepo as String,
                 data.git.releaseManagerBranch as String,
+                DEFAULT_BRANCH_RELEASE_MANAGER,
                 tmpFolder + "/releasemanager")
 
         this.data.metadata = loadMetadata(tmpFolder + "/releasemanager")
@@ -136,8 +142,7 @@ class ProjectData {
     private Map loadMetadata(String workspace){
         Map result = parseMetadataFile(workspace)
         result.description = (result.description)?: ""
-        result.repositories = (result.repositories)?: ""
-        updateRepositories(result)
+        result.repositories = loadRepositories(result.repositories)
         result.capabilities = (result.capabilities )?: []
         updateLevaDocCapability(result)
         result.environments = (result.environments)?: [:]
@@ -167,37 +172,51 @@ class ProjectData {
         }
     }
 
-    private void updateRepositories(Map result) {
-        result.repositories.eachWithIndex { repo, index ->
-            // Check for existence of required attribute 'repositories[i].id'
-            if (!repo.id?.trim()) {
-                throw new IllegalArgumentException(
-                        "Error: unable to parse project meta data. Required attribute 'repositories[${index}].id' is undefined.")
-            }
-
-            repo.data = [
-                    openshift: [:],
-                    documents: [:],
-            ]
-
-            // Set repo type, if not provided
-            if (!repo.type?.trim()) {
-                repo.type = PipelineConfig.REPO_TYPE_ODS_CODE
-            }
-
-            repo.metadata = loadMetadataRepo(repo)
+    private loadRepositories(List repositories) {
+        List<RepoData> reposData = []
+        for(Map repo : repositories) {
+            def repoMetadata = loadRepoMetadataFromRepo(repo)
+            RepoData repoData = new RepoData(repo, repoMetadata)
+            reposData.add(repoData)
         }
+        return reposData
     }
 
-    private Map<String, String> loadMetadataRepo(repo) {
-        return  [
+    private RepoMetadata loadRepoMetadataFromRepo(repo) {
+        if ((! repo) || (! repo.id)) {
+            throw new RuntimeException("Repository id cannot be blank or null.")
+        }
+        String projectId = data.projectId as String
+        String repoId = "${projectId}-${repo.id}"
+        String branch = data.git.releaseManagerBranch as String
+        String targetFolder = tmpFolder + "/" + repoId
+
+        String defaultBranch = repo.branch ?: "master"
+        bitbucketService.downloadRepoMetadata(projectId, repoId, branch, defaultBranch, targetFolder)
+        Map repoMetadata = parseRepoMetadataFile(targetFolder)
+        log.debug(prettyPrint(toJson(repoMetadata)))
+
+        String gitUrl = bitbucketService.buildRepositoryUrl(projectId, repoId)
+        return new RepoMetadata([
                 id: repo.id,
-                name: repo.name,
-                description: "myDescription-A",
-                supplier: "mySupplier-A",
-                version: "myVersion-A",
-                references: "myReferences-A"
-        ]
+                name: repoMetadata.name,
+                description: repoMetadata.description,
+                supplier: repoMetadata.supplier,
+                version: repoMetadata.version,
+                references: repoMetadata.references,
+                gitUrl: gitUrl
+        ])
+    }
+
+    private Map parseRepoMetadataFile(String repoFolder) {
+        String filename = METADATA_FILE_NAME
+        def file = Paths.get(repoFolder, filename).toFile()
+        if (!file.exists()) {
+            throw new RuntimeException("Error: unable to load project meta data. File '${repoFolder}/${filename}' does not exist.")
+        }
+
+        Map result = new Yaml().load(file.text)
+        return result
     }
 
     private Map parseMetadataFile(String workspace) {
@@ -467,7 +486,7 @@ class ProjectData {
         return this.data.metadata.name
     }
 
-    List<Map> getRepositories() {
+    List<RepoData> getRepositories() {
         return this.data.metadata.repositories
     }