Release notes automation (#4196)

Signed-off-by: Sachin Sahu <sachinksahu.431@gmail.com>

.gitignore

@@ -37,3 +37,4 @@ out.txt
 vars/
 
 test-report.yml
+src/release_notes_workflow/results/*

Pipfile

@@ -38,6 +38,7 @@ typed-ast = "~=1.5.4"
 zipp = "~=3.8.1"
 importlib-metadata = "~=4.12.0"
 ruamel-yaml = "~=0.17.21"
+mistune = "~=3.0.1"
 
 [dev-packages]
 

src/release_notes_workflow/README.md

@@ -1,13 +1,19 @@
-#### Components Release Notes Check 
+- [Consolidated Release Notes Automation](#consolidated-release-notes-automation)
+  - [Components Release Notes Check](#components-release-notes-check)
+  - [Consolidated Release Notes](#consolidated-release-notes)
+
+## Consolidated Release Notes
+
+### Components Release Notes Check 
 
 Pulls the latest code to check if the release notes exists and whether new commits have been made based on user passed argument `--date`. Outputs a formated markdown table as follows.
 
-*Usage*
+#### Usage
 ```
 ./release_notes.sh check manifests/3.0.0/opensearch-3.0.0.yml --date 2022-07-26
 ```
 
-*Sample Output*
+#### Sample Output
 ```
 #  OpenSearch CommitID(after 2022-07-26) & Release Notes info
 |          Repo           |   Branch   |CommitID|Commit Date|Release Notes|
@@ -25,7 +31,7 @@ Pulls the latest code to check if the release notes exists and whether new commi
 
 The workflow uses the following arguments:
 * `--date`: To check if commit exists after a specific date (in format yyyy-mm-dd, example 2022-07-26).
-* `--output`: To dump the output into an `.md` file, example `--output table.md`).
+* `--output`: To dump the output into an `.md` file, example `--output table.md`.
 
 
 The following options are available.
@@ -35,3 +41,61 @@ The following options are available.
 | --date             | Shows commit after a specific date.                                     |
 | --output           | Saves the table output to user specified file.  		       	       |
 | -v, --verbose      | Show more verbose output.                                               |
+
+### Consolidated Release Notes
+
+This workflow generates a consolidated release notes for all the components. 
+It utilizes the output from the preceding step to compile these consolidated release notes. If the preceding step hasn't been executed, it will automatically run that step first before generating the consolidated release notes. 
+
+#### Usage
+```
+./release_notes.sh compile manifests/3.0.0/opensearch-3.0.0.yml --date 2022-07-26
+```
+
+#### Sample Output
+Two output files are generated:
+- Markdown table containing links to individual components' release notes for quick reference (Example: `release_notes_table-2.10.0.md`)
+- Consolidated release notes for all the components (Example: `release_notes-2.10.0.md`)
+
+<details> 
+<summary>Markdown table with links</summary>
+
+#  OpenSearch CommitID(after 2022-07-26) & Release Notes info
+|          Repo           |    Branch     |CommitID|Commit Date|Release Notes Exists|                                                                                URL                                                                                 |
+|-------------------------|---------------|--------|-----------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|alerting                 |[tags/2.10.0.0]|dc1b9bf |2023-09-18 |True                |https://raw.githubusercontent.com/opensearch-project/alerting/2.10.0.0/release-notes/opensearch-alerting.release-notes-2.10.0.0.md                                  |
+|anomaly-detection        |[tags/2.10.0.0]|bc4d8b1 |2023-09-08 |True                |https://raw.githubusercontent.com/opensearch-project/anomaly-detection/2.10.0.0/release-notes/opensearch-anomaly-detection.release-notes-2.10.0.0.md                |
+|asynchronous-search      |[tags/2.10.0.0]|a312d9a |2023-09-07 |True                |https://raw.githubusercontent.com/opensearch-project/asynchronous-search/2.10.0.0/release-notes/opensearch-asynchronous-search.release-notes-2.10.0.0.md            |
+|common-utils             |[tags/2.10.0.0]|0352c2f |2023-09-08 |True                |https://raw.githubusercontent.com/opensearch-project/common-utils/2.10.0.0/release-notes/opensearch-common-utils.release-notes-2.10.0.0.md                          |
+|cross-cluster-replication|[tags/2.10.0.0]|dee2f60 |2023-09-08 |True                |https://raw.githubusercontent.com/opensearch-project/cross-cluster-replication/2.10.0.0/release-notes/opensearch-cross-cluster-replication.release-notes-2.10.0.0.md|
+|custom-codecs            |[tags/2.10.0.0]|3437b43 |2023-09-15 |True                |https://raw.githubusercontent.com/opensearch-project/custom-codecs/2.10.0.0/release-notes/opensearch-custom-codecs.release-notes-2.10.0.0.md                        |
+|geospatial               |[tags/2.10.0.0]|a3da222 |2023-09-12 |True                |https://raw.githubusercontent.com/opensearch-project/geospatial/2.10.0.0/release-notes/opensearch-geospatial.release-notes-2.10.0.0.md                              |
+|index-management         |[tags/2.10.0.0]|062badd |2023-09-07 |True                |https://raw.githubusercontent.com/opensearch-project/index-management/2.10.0.0/release-notes/opensearch-index-management.release-notes-2.10.0.0.md                  |
+|job-scheduler            |[tags/2.10.0.0]|e9d3637 |2023-09-12 |True                |https://raw.githubusercontent.com/opensearch-project/job-scheduler/2.10.0.0/release-notes/opensearch.job-scheduler.release-notes-2.10.0.0.md                        |
+|k-NN                     |[tags/2.10.0.0]|e437016 |2023-09-07 |True                |https://raw.githubusercontent.com/opensearch-project/k-NN/2.10.0.0/release-notes/opensearch-knn.release-notes-2.10.0.0.md                                           |
+|ml-commons               |[tags/2.10.0.0]|521214b |2023-09-13 |True                |https://raw.githubusercontent.com/opensearch-project/ml-commons/2.10.0.0/release-notes/opensearch-ml-common.release-notes-2.10.0.0.md                               |
+|neural-search            |[tags/2.10.0.0]|9476d43 |2023-09-07 |True                |https://raw.githubusercontent.com/opensearch-project/neural-search/2.10.0.0/release-notes/opensearch-neural-search.release-notes-2.10.0.0.md                        |
+|notifications            |[tags/2.10.0.0]|0a9dfb0 |2023-09-07 |True                |https://raw.githubusercontent.com/opensearch-project/notifications/2.10.0.0/release-notes/opensearch-notifications.release-notes-2.10.0.0.md                        |
+|opensearch-observability |[tags/2.10.0.0]|d2c087c |2023-09-13 |True                |https://raw.githubusercontent.com/opensearch-project/observability/2.10.0.0/release-notes/opensearch-observability.release-notes-2.10.0.0.md                        |
+|opensearch-reports       |[tags/2.10.0.0]|3095e3c |2023-09-13 |True                |https://raw.githubusercontent.com/opensearch-project/reporting/2.10.0.0/release-notes/opensearch-reporting.release-notes-2.10.0.0.md                                |
+|performance-analyzer     |[tags/2.10.0.0]|3ee56fc |2023-09-07 |True                |https://raw.githubusercontent.com/opensearch-project/performance-analyzer/2.10.0.0/release-notes/opensearch-performance-analyzer.release-notes-2.10.0.0.md          |
+|security                 |[tags/2.10.0.0]|6daa697 |2023-09-12 |True                |https://raw.githubusercontent.com/opensearch-project/security/2.10.0.0/release-notes/opensearch-security.release-notes-2.10.0.0.md                                  |
+|security-analytics       |[tags/2.10.0.0]|e005b5a |2023-09-19 |True                |https://raw.githubusercontent.com/opensearch-project/security-analytics/2.10.0.0/release-notes/opensearch-security-analytics.release-notes-2.10.0.0.md              |
+|sql                      |[tags/2.10.0.0]|ef18b38 |2023-09-07 |True                |https://raw.githubusercontent.com/opensearch-project/sql/2.10.0.0/release-notes/opensearch-sql.release-notes-2.10.0.0.md                                            |
+
+</details>
+
+---
+
+The workflow uses the following arguments:
+* `--date`: To check if commit exists after a specific date (in format yyyy-mm-dd, example 2022-07-26). This is optional if the previous step is already run.
+* `--output`: To dump the consolidated release notes into an `.md` file, example `--output table.md`.
+
+
+The following options are available.
+
+| name               | description                                                             |
+|--------------------|-------------------------------------------------------------------------|
+| --date             | Shows commit after a specific date.                                     |
+| --output           | Saves the release notes to user specified file.  		       	       |
+| -v, --verbose      | Show more verbose output.                                               |

src/release_notes_workflow/release_notes.py

@@ -19,18 +19,32 @@
 
 class ReleaseNotes:
 
-    def __init__(self, manifest: InputManifest, date: str) -> None:
+    def __init__(self, manifest: InputManifest, date: str, action_type: str) -> None:
         self.manifest = manifest
         self.date = date
+        self.action_type = action_type
 
     def table(self) -> MarkdownTableWriter:
         table_result = []
         for component in self.manifest.components.select():
+            if component.name == 'OpenSearch' or component.name == 'OpenSearch-Dashboards' or component.name == 'notifications-core':
+                continue
             if hasattr(component, "repository"):
                 table_result.append(self.check(component))  # type: ignore[arg-type]
+
+        # Sort table_result based on Repo column
+        table_result.sort(key=lambda x: (x[0], x[1]) if len(x) > 1 else x[0])
+
+        if self.action_type == "check":
+            headers = ["Repo", "Branch", "CommitID", "Commit Date", "Release Notes Exists"]
+        elif self.action_type == "compile":
+            headers = ["Repo", "Branch", "CommitID", "Commit Date", "Release Notes Exists", "URL"]
+        else:
+            raise ValueError("Invalid action_type. Use 'check' or 'compile'.")
+
         writer = MarkdownTableWriter(
             table_name=f" {self.manifest.build.name} CommitID(after {self.date}) & Release Notes info",
-            headers=["Repo", "Branch", "CommitID", "Commit Date", "Release Notes"],
+            headers=headers,
             value_matrix=table_result
         )
         return writer
@@ -57,4 +71,13 @@ def check(self, component: InputComponentFromSource) -> List:
                     results.append(None)
                     results.append(None)
                 results.append(release_notes.exists())
+
+                if(release_notes.exists()):
+                    releasenote = os.path.basename(release_notes.full_path)
+                    repo_name = component.repository.split("/")[-1].split('.')[0]
+                    repo_ref = component.ref.split("/")[-1]
+                    url = f"https://raw.githubusercontent.com/opensearch-project/{repo_name}/{repo_ref}/release-notes/{releasenote}"
+                    results.append(url)
+                else:
+                    results.append(None)
         return results

src/release_notes_workflow/release_notes_check_args.py

@@ -19,7 +19,7 @@ class ReleaseNotesCheckArgs:
 
     def __init__(self) -> None:
         parser = argparse.ArgumentParser(description="Checkout an OpenSearch Bundle and check for CommitID and Release Notes")
-        parser.add_argument("action", choices=["check"], help="Operation to perform.")
+        parser.add_argument("action", choices=["check", "compile"], help="Operation to perform.")
         parser.add_argument("manifest", type=argparse.FileType("r"), help="Manifest file.")
         parser.add_argument(
             "-v",

src/release_notes_workflow/release_notes_component.py

@@ -25,27 +25,44 @@ def filename(self) -> str:
 
     @property
     def path(self) -> str:
-        return os.path.join(self.root, "release-notes")
+        release_notes_path = os.path.join(self.root, "release-notes")
+        return release_notes_path
+
+    # combine path with the file in files_in_path such that it ends with the filename
+    @property
+    def full_path(self) -> str:
+        files_in_path = os.listdir(self.path)
+        for fname in files_in_path:
+            if fname.endswith(self.filename):
+                release_notes_full_path = os.path.join(self.path, fname)
+                return release_notes_full_path
+        return None
 
     def path_exists(self) -> bool:
-        return os.path.exists(self.path)
+        path_exists = os.path.exists(self.path)
+        return path_exists
 
     def exists(self) -> bool:
-        return self.path_exists() and any(fname.endswith(self.filename) for fname in os.listdir(self.path))
+        if not os.path.exists(self.path):
+            return False
+        files_in_path = os.listdir(self.path)
+        return self.path_exists() and any(fname.endswith(self.filename) for fname in files_in_path)
 
 
 class ReleaseNotesOpenSearch(ReleaseNotesComponent):
 
     @property
     def filename(self) -> str:
-        return f'.release-notes-{self.build_version}.md'
+        release_notes_filename = f'.release-notes-{self.build_version}.md'
+        return release_notes_filename
 
 
 class ReleaseNotesOpenSearchPlugin(ReleaseNotesComponent):
 
     @property
     def filename(self) -> str:
-        return f'.release-notes-{self.build_version}.0.md'
+        release_notes_filename = f'.release-notes-{self.build_version}.0.md'
+        return release_notes_filename
 
 
 class ReleaseNotesComponents: