Merge pull request #102 from caponetto/cherry-picks-from-upstream2

Cherry-pick improvements from upstream

README.md

@@ -26,8 +26,8 @@ limitations under the License.
 [![Gitter](https://badges.gitter.im/elyra-ai/community.svg)](https://gitter.im/elyra-ai/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
 
-### Note: 
-*This repository/project is for opendatahub-io/Data-science-pipelines v2 and workbench work only.*  
+### Note:
+*This repository/project is for opendatahub-io/Data-science-pipelines v2 and workbench work only.*
 Please refer to upstream [elyra](https://github.com/elyra-ai/elyra) for stable version.
 
 # Elyra

binder/postBuild

@@ -42,19 +42,19 @@ set -x
 
 # repo2docker clones the Elyra repository before the Docker image is built
 # and uses the cloned directory as "root". The git extension recognizes that
-# the directory is git-enabled and therefore prevents the user from 
+# the directory is git-enabled and therefore prevents the user from
 # cloning other repositories using the UI. Remove the git artifacts in the
-# root directory to work around this issue. 
+# root directory to work around this issue.
 rm -rf ~/.git*
 
 # Add getting started assets to Docker image
 # (1) Clone the examples repository.
-git clone https://github.com/elyra-ai/examples.git 
+git clone https://github.com/elyra-ai/examples.git
 
 EXAMPLES_BASE_DIR=examples
 
 # (2) Copy all getting started assets to the working directory,
-#     which is referenced when binder is invoked, e.g. 
+#     which is referenced when binder is invoked, e.g.
 #     https://mybinder.org/v2/gh/elyra-ai/elyra/main?urlpath=lab/tree/binder-demo
 cp -r $EXAMPLES_BASE_DIR/binder/getting-started/* binder-demo/
 

docs/requirements.txt

@@ -1,6 +1,7 @@
 sphinx>=1.8
 sphinx_rtd_theme==2.0.0
 recommonmark>=0.6.0
+sphinxcontrib-googleanalytics>=0.4
 sphinx-markdown-tables>=0.0.16
 sphinx-version-warning
 markdown>=3.4.1

docs/source/conf.py

@@ -55,6 +55,7 @@
 extensions = [
     "sphinx_rtd_theme",
     "recommonmark",
+    "sphinxcontrib.googleanalytics",
     "sphinx_markdown_tables",
     "versionwarning.extension",
 ]
@@ -81,9 +82,7 @@
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-html_theme_options = {
-    "analytics_id": "UA-169426804-1",
-}
+googleanalytics_id = "UA-169426804-1"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,

docs/source/getting_started/changelog.md

@@ -2,9 +2,82 @@
 
 A summary of new feature highlights is located on the [GitHub release page](https://github.com/elyra-ai/elyra/releases).
 
-## Release 4.0.0
-
-- Add functionality to select resource limits from the GUI [#3202](https://github.com/elyra-ai/elyra/pull/3202)
+## Release 4.0.0rc0 - 01/03/2025
+
+Our 4.0.0 release candidate 0 includes the following high-level changes
+
+- Add support for JupyterLab 4.2.5 and above - [#3201](https://github.com/elyra-ai/elyra/pull/3201)
+- Enable conditional install of runtime and it's dependencies (e.g: KFP, Airflow) - [#3248](https://github.com/elyra-ai/elyra/pull/3248)
+- Add Airflow 2.x support for generic pipelines and generic components - [#3167](https://github.com/elyra-ai/elyra/pull/3167)
+- Multiple dependency updates with security fixes
+
+See the full changelog below:
+
+- Force dist metadata-version to avoid pypi upload issues
+- Show validation warnings only after the first form submission - [#87](https://github.com/elyra-ai/elyra/pull/87)
+- Revert "Remove read_entry_data method from class ComponentCatalogConnector - [#3141](https://github.com/elyra-ai/elyra/pull/3141)
+- Remove obsolete read_entry_data method from class ComponentCatalogConnector - [#3141](https://github.com/elyra-ai/elyra/pull/3141)
+- Airflow 2 >= 2.7.0 generic pipeline/component support. See also #3166. - [#3167](https://github.com/elyra-ai/elyra/pull/3167)
+- Update copyright to 2025 - [#3256](https://github.com/elyra-ai/elyra/pull/3256)
+- Update repository readme and documentation with JupyterLab v4 information - [#3145](https://github.com/elyra-ai/elyra/pull/3145)
+- Bump jinja2 from 3.1.4 to 3.1.5 in /etc/generic - [#3261](https://github.com/elyra-ai/elyra/pull/3261)
+- Bump cross-spawn from 6.0.5 to 6.0.6 - [#3262](https://github.com/elyra-ai/elyra/pull/3262)
+- Refactor pipeline implementation to use common pattern for file naming - [#3160](https://github.com/elyra-ai/elyra/pull/3160)
+- Migrate Elyra extensions to support JupyterLab 4.x - [#3201](https://github.com/elyra-ai/elyra/pull/3201)
+- Fix Python formatting to get rid of linter errors from `make lint-server` - [#3260](https://github.com/elyra-ai/elyra/pull/3260)
+- Use Python instead of curl to download test file on validate-runtime-image target - [#3257](https://github.com/elyra-ai/elyra/pull/3257)
+- Update runtime images with latest component versions - [#3255](https://github.com/elyra-ai/elyra/pull/3255)
+- Bump follow-redirects from 1.15.2 to 1.15.9 - [#3252](https://github.com/elyra-ai/elyra/pull/3252)
+- Bump nanoid from 3.3.4 to 3.3.8 - [#3254](https://github.com/elyra-ai/elyra/pull/3254)
+- Bump tornado from 6.4.1 to 6.4.2 in /etc/generic - [#3253](https://github.com/elyra-ai/elyra/pull/3253)
+- Make KFP dependency optional in build and during runs - [#3248](https://github.com/elyra-ai/elyra/pull/3248)
+- Pin jupyterlab_pygments to fix build issues - [#3250](https://github.com/elyra-ai/elyra/pull/3250)
+- Update Sphinx support for Google Analytics - [#3251](https://github.com/elyra-ai/elyra/pull/3251)
+- Bump elliptic from 6.5.7 to 6.6.0 - [#3249](https://github.com/elyra-ai/elyra/pull/3249)
+- Add additional readthedocs configs - [#3247](https://github.com/elyra-ai/elyra/pull/3247)
+- Remove conda tools from readthedocs build - [#3246](https://github.com/elyra-ai/elyra/pull/3246)
+- Update readthedoc configuration to address build failures - [#3245](https://github.com/elyra-ai/elyra/pull/3245)
+- Document system-level environment variables configs - [#3243](https://github.com/elyra-ai/elyra/pull/3243)
+- Bump webpack from 5.76.0 to 5.94.0 - [#3244](https://github.com/elyra-ai/elyra/pull/3244)
+- Force operation output to S3 based on env var config - [#3227](https://github.com/elyra-ai/elyra/pull/3227)
+- Bump elliptic from 6.5.4 to 6.5.7 - [#3242](https://github.com/elyra-ai/elyra/pull/3242)
+- Dependency updates including nbconvert version without lxml - [#3240](https://github.com/elyra-ai/elyra/pull/3240)
+- Allow users to include ssl_sa_certs via env vars for kfp_tekton client - [#3150](https://github.com/elyra-ai/elyra/pull/3150)
+- Bump tornado from 6.3.3 to 6.4.1 in /etc/generic - [#3234](https://github.com/elyra-ai/elyra/pull/3234)
+- Bump urllib3 from 1.26.18 to 1.26.19 in /etc/generic - [#3236](https://github.com/elyra-ai/elyra/pull/3236)
+- Bump tar from 6.1.11 to 6.2.1 - [#3231](https://github.com/elyra-ai/elyra/pull/3231)
+- Bump ejs from 3.1.8 to 3.1.10 - [#3230](https://github.com/elyra-ai/elyra/pull/3230)
+- Bump jinja2 from 3.0.3 to 3.1.4 in /etc/generic - [#3229](https://github.com/elyra-ai/elyra/pull/3229)
+- Bump requests from 2.31.0 to 2.32.0 in /etc/generic - [#3232](https://github.com/elyra-ai/elyra/pull/3232)
+- Fixes validation when there are pipeline params and default properties - [#3195](https://github.com/elyra-ai/elyra/pull/3195)
+- Bump jinja2 from 3.0.3 to 3.1.3 in /etc/generic - [#3207](https://github.com/elyra-ai/elyra/pull/3207)
+- Bump es5-ext from 0.10.62 to 0.10.63 - [#3218](https://github.com/elyra-ai/elyra/pull/3218)
+- Bump ip from 1.1.8 to 1.1.9 - [#3216](https://github.com/elyra-ai/elyra/pull/3216)
+- Bump @babel/traverse from 7.19.1 to 7.23.6 - [#3200](https://github.com/elyra-ai/elyra/pull/3200)
+- Bump browserify-sign from 4.2.1 to 4.2.2 - [#3193](https://github.com/elyra-ai/elyra/pull/3193)
+- Enable supporting other docker clients such as podman via build config - [#3226](https://github.com/elyra-ai/elyra/pull/3226)
+- Fix broken code of conduct link - [#3213](https://github.com/elyra-ai/elyra/pull/3213)
+- Fix linting issues breaking CI - [#3215](https://github.com/elyra-ai/elyra/pull/3215)
+- Modify version range of wrapt to fix Python 3.11 build issue - [#3197](https://github.com/elyra-ai/elyra/pull/3197)
+- Remove deprecated install command from elyra-metadata - [#3143](https://github.com/elyra-ai/elyra/pull/3143)
+- Editing files to add resource requests and limits - [#3202](https://github.com/elyra-ai/elyra/pull/3202)
+- Skip minio-7.2.1 to avoid urllib version mismatch - [#3204](https://github.com/elyra-ai/elyra/pull/3204)
+- Fix CI Build by temporarily disabling Python 3.11 build - [#3198](https://github.com/elyra-ai/elyra/pull/3198)
+- Bump urllib3 from 1.26.17 to 1.26.18 in /etc/generic - [#3191](https://github.com/elyra-ai/elyra/pull/3191)
+- Update the broken link for deploying opendatahub - [#3188](https://github.com/elyra-ai/elyra/pull/3188)
+- Bump urllib3 from 1.26.9 to 1.26.17 in /etc/generic - [#3187](https://github.com/elyra-ai/elyra/pull/3187)
+- Bump tornado from 6.3.2 to 6.3.3 in /etc/generic - [#3178](https://github.com/elyra-ai/elyra/pull/3178)
+- Bump word-wrap from 1.2.3 to 1.2.5 - [#3185](https://github.com/elyra-ai/elyra/pull/3185)
+- Bump semver from 5.7.1 to 5.7.2 - [#3171](https://github.com/elyra-ai/elyra/pull/3171)
+- Cap jupyter-resource-usage to avoid Lab 4.0 dependencies - [#3184](https://github.com/elyra-ai/elyra/pull/3184)
+- Update copyright doc config - [#3163](https://github.com/elyra-ai/elyra/pull/3163)
+- Bump requests from 2.27.1 to 2.31.0 in /etc/generic - [#3156](https://github.com/elyra-ai/elyra/pull/3156)
+- Update Copyright year - [#3162](https://github.com/elyra-ai/elyra/pull/3162)
+- Bump tornado from 6.1 to 6.3.2 in /etc/generic - [#3158](https://github.com/elyra-ai/elyra/pull/3158)
+- Convert --valid-only to --include-invalid on list subcommand - [#3142](https://github.com/elyra-ai/elyra/pull/3142)
+- Update pytorch runtime to fix validate images check - [#3148](https://github.com/elyra-ai/elyra/pull/3148)
+- Update ODH install instructions - [#3146](https://github.com/elyra-ai/elyra/pull/3146)
+- Add link to 3.15.0 new feature highlight blog post - [#3121](https://github.com/elyra-ai/elyra/pull/3121)
 
 ## Release 3.15.0 - 03/28/2023
 
@@ -690,7 +763,7 @@ High level enhancements
 - Integrate new content parser with support for refreshing environment vars from
   notebooks and scripts
 - New Pipeline validation service
-- Support for KFP 1.6.3 / Tekton 0.8.1 
+- Support for KFP 1.6.3 / Tekton 0.8.1
 - Fix DEX authentication issues
 - Upgraded to new Pipeline Editor version 1.0.0 and Elyra Canvas to version 11.0.0
 - Update to JupyterLab LSP 3.8.0 and migrate to python-lsp-server as the Python language server
@@ -1080,7 +1153,7 @@ High level enhancements
 - Add support for JupyterLab 3.x
 - Add new Language Server Protocol (LSP) capabilities
 - Add support for Kubeflow Pipelines using Argo and Tekton engines
-- Publishing Elyra images to both docker.io and quay.io  
+- Publishing Elyra images to both docker.io and quay.io
 - Multiple updates to overall documentation
 - Increased test coverage
 

docs/source/recipes/running-elyra-in-air-gapped-environment.md

@@ -30,7 +30,7 @@ These dependencies are in addition to any dependencies you wish to include in th
 
 ## Runtime dependencies
 
-When using Elyra's features to [build, export, or run pipelines](../user_guide/pipelines.md), additional runtime dependencies must be accessible in the environment where JupyterLab is installed and the Kubernetes cluster where the pipeline runtime environment (Kubeflow Pipelines or Apache Airflow) is installed. 
+When using Elyra's features to [build, export, or run pipelines](../user_guide/pipelines.md), additional runtime dependencies must be accessible in the environment where JupyterLab is installed and the Kubernetes cluster where the pipeline runtime environment (Kubeflow Pipelines or Apache Airflow) is installed.
 
 ![runtime dependencies](../images/recipes/running-elyra-in-air-gapped-environment/runtime-dependencies.png)
 
@@ -39,9 +39,9 @@ In the chart above the arrows indicate whether read access, write access, or bot
 ### JupyterLab environment dependencies
 
 Elyra requires access to the following dependencies when you build, export, or submit a pipeline:
-- **Runtime environment**: Elyra requires access to the Kubernetes cluster where Kubeflow Pipelines or Apache Airflow is running. 
+- **Runtime environment**: Elyra requires access to the Kubernetes cluster where Kubeflow Pipelines or Apache Airflow is running.
 - **GitHub repository or GitLab project:** For Apache Airflow Elyra requires access to the GitHub repository or GitLab project that is configured in the [runtime configuration](../user_guide/runtime-conf.html#git-type-git-type).
-- **Component definitions for [custom components](../user_guide/pipeline-components.html#custom-components)**: Elyra utilizes [catalog connectors](../user_guide/pipeline-components.html#component-catalogs) to locate and load component definitions. The connectors must be able to communicate with the configured catalog, or you will not be able to submit or export pipelines. For example, if a pipeline utilizes a component that is stored in a URL component catalog, the component's URL must be accessible via an anonymous HTTP request.  
+- **Component definitions for [custom components](../user_guide/pipeline-components.html#custom-components)**: Elyra utilizes [catalog connectors](../user_guide/pipeline-components.html#component-catalogs) to locate and load component definitions. The connectors must be able to communicate with the configured catalog, or you will not be able to submit or export pipelines. For example, if a pipeline utilizes a component that is stored in a URL component catalog, the component's URL must be accessible via an anonymous HTTP request.
 - **S3-compatible cloud storage for [generic components](../user_guide/pipeline-components.html#generic-components)**: During pipeline [export](../user_guide/pipelines.html#exporting-pipelines) or [submission](../user_guide/pipelines.html#running-pipelines) Elyra uploads pipeline artifacts to an S3 bucket. These artifacts are downloaded to the pipeline runtime environment when the pipeline is executed.
 
 ### Runtime environment dependencies
@@ -63,7 +63,7 @@ During pipeline execution in the Kubeflow Pipelines or Apache Airflow environmen
     - For Apache Airflow:
         - `ELYRA_BOOTSTRAP_SCRIPT_URL` (URL of `.../elyra/airflow/bootstrapper.py`)
         - `ELYRA_REQUIREMENTS_URL` (URL of `.../etc/generic/requirements-elyra.txt`)
-    
+
   Alternatively, for Kubeflow Pipeline users can store the bootstrapper.py, pip.conf and requirements-elyra.txt files in the runtime container itself, so it could be used without having to set the above variables. The path to these files needs to be configured, so Elyra execution can pick it up.
-    - `ELYRA_FILE_BASE_PATH` (default `/opt/app-root/bin/utils`) 
+    - `ELYRA_FILE_BASE_PATH` (default `/opt/app-root/bin/utils`)
 - **S3-compatible cloud storage for [generic components](../user_guide/pipeline-components.html#generic-components)**: When processing pipeline nodes that are implemented using [generic components](../user_guide/pipeline-components.html#generic-components), Elyra downloads the pipeline artifacts that were uploaded when the pipeline was exported or submitted.

elyra/_version.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-__version__ = "0.0.0.dev"
+__version__ = "4.0.0.dev0"

elyra/util/http.py

@@ -61,9 +61,9 @@ def write_error(self, status_code, **kwargs):
                     if isinstance(exception.args[0], Exception):
                         reply["message"] = (
                             f"Error. The server sent an invalid response.\
-                            \nPlease open an issue and provide this error message,\
-                            any error details, and any related JupyterLab log messages.\
-                            \n\nError found:\n{str(exception.args[0])}"
+                                \nPlease open an issue and provide this error message,\
+                                any error details, and any related JupyterLab log messages.\
+                                \n\nError found:\n{str(exception.args[0])}"
                         )
                     else:
                         reply["message"] = str(exception.args[0])

etc/generic/requirements-elyra.txt

@@ -2,7 +2,7 @@
 ipykernel==6.25.2
 ipython==8.12.3
 ipython-genutils==0.2.0
-jinja2==3.1.4
+jinja2==3.1.5
 jupyter-client==7.4.9
 jupyter-core==5.3.1
 MarkupSafe==2.1.1
@@ -14,7 +14,7 @@ papermill==2.6.0
 pyzmq==24.0.1
 prompt-toolkit==3.0.43
 requests==2.32.3
-tornado==6.4.1
+tornado==6.4.2
 traitlets==5.10.0
 urllib3==1.26.19
 #

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.0-dev",
+  "version": "4.0.0-dev",
   "npmClient": "yarn",
   "command": {
     "publish": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "elyra",
-  "version": "0.0.0-dev",
+  "version": "4.0.0-dev",
   "private": true,
   "workspaces": {
     "packages": [

packages/code-snippet/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elyra/code-snippet-extension",
-  "version": "0.0.0-dev",
+  "version": "4.0.0-dev",
   "description": "JupyterLab extension - Reusable code snippets for your Notebook and Python Scripts",
   "keywords": [
     "jupyter",
@@ -53,9 +53,9 @@
     "watch:labextension": "jupyter labextension watch ."
   },
   "dependencies": {
-    "@elyra/metadata-common": "0.0.0-dev",
-    "@elyra/services": "0.0.0-dev",
-    "@elyra/ui-components": "0.0.0-dev",
+    "@elyra/metadata-common": "4.0.0-dev",
+    "@elyra/services": "4.0.0-dev",
+    "@elyra/ui-components": "4.0.0-dev",
     "@jupyterlab/application": "^4.2.5",
     "@jupyterlab/apputils": "^4.2.5",
     "@jupyterlab/builder": "^4.2.5",

packages/code-viewer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elyra/code-viewer-extension",
-  "version": "0.0.0-dev",
+  "version": "4.0.0-dev",
   "description": "JupyterLab extension - Display code without a file",
   "keywords": [
     "jupyter",

packages/metadata-common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elyra/metadata-common",
-  "version": "0.0.0-dev",
+  "version": "4.0.0-dev",
   "description": "JupyterLab - Widgets for interacting with metadata",
   "keywords": [
     "jupyter",
@@ -52,8 +52,8 @@
     "watch:labextension": "jupyter labextension watch ."
   },
   "dependencies": {
-    "@elyra/services": "0.0.0-dev",
-    "@elyra/ui-components": "0.0.0-dev",
+    "@elyra/services": "4.0.0-dev",
+    "@elyra/ui-components": "4.0.0-dev",
     "@jupyterlab/application": "^4.2.5",
     "@jupyterlab/apputils": "^4.2.5",
     "@jupyterlab/codeeditor": "^4.2.5",

packages/metadata/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elyra/metadata-extension",
-  "version": "0.0.0-dev",
+  "version": "4.0.0-dev",
   "description": "JupyterLab extension - View and edit metadata",
   "keywords": [
     "jupyter",
@@ -53,8 +53,8 @@
     "watch:labextension": "jupyter labextension watch ."
   },
   "dependencies": {
-    "@elyra/metadata-common": "0.0.0-dev",
-    "@elyra/services": "0.0.0-dev",
+    "@elyra/metadata-common": "4.0.0-dev",
+    "@elyra/services": "4.0.0-dev",
     "@jupyterlab/application": "^4.2.5",
     "@jupyterlab/apputils": "^4.2.5",
     "@jupyterlab/builder": "^4.2.5",

packages/pipeline-editor/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elyra/pipeline-editor-extension",
-  "version": "0.0.0-dev",
+  "version": "4.0.0-dev",
   "description": "JupyterLab extension - Visual editor to build Notebook pipelines",
   "keywords": [
     "jupyter",
@@ -54,11 +54,11 @@
     "watch:labextension": "jupyter labextension watch ."
   },
   "dependencies": {
-    "@elyra/metadata-common": "0.0.0-dev",
+    "@elyra/metadata-common": "4.0.0-dev",
     "@elyra/pipeline-editor": "1.12.1",
     "@elyra/pipeline-services": "1.12.1",
-    "@elyra/services": "0.0.0-dev",
-    "@elyra/ui-components": "0.0.0-dev",
+    "@elyra/services": "4.0.0-dev",
+    "@elyra/ui-components": "4.0.0-dev",
     "@jupyterlab/application": "^4.2.5",
     "@jupyterlab/apputils": "^4.2.5",
     "@jupyterlab/builder": "^4.2.5",