Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial changes to support runtime sub package installs #2259

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

Initial changes to support runtime sub package installs #2259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0dfb24f
Initial changes to support runtime sub package installs
akchinSTC Oct 27, 2021
b2c7e22
Fix typos in metadata
akchinSTC Apr 6, 2022
0531265
Load processor only if library is found
akchinSTC Apr 11, 2022
3a30545
Remove test deps no longer needed
akchinSTC Apr 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ dist/
downloads/
eggs/
.eggs/
examples
lib/
lib64/
parts/
Expand Down
24 changes: 19 additions & 5 deletions elyra/metadata/schemasproviders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,20 @@
# We may not have kfp-tekton available and that's okay!
TektonClient = None

try:
from kfp import compiler # noqa
except ImportError:
# KFP python package is not installed
compiler = None

try:
# Check to see if Airflow package is installed, since we do not have any dependencies
# on any Airflow package, we use GitHub as our canary
from github import Github # noqa
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this will be sufficient. Won't we still need the ability for the user to physically indicate their intentions? Like pip install elyra[kfp] or pip install elyra[airflow].

Do we need a dummy airflow package that we could use (and install via the previous example) to make the user's intention known to server code? This package, while on PyPi and Conda-forge, would likely never have to change, so I'm not sure there would be much of a burden other than its initial creation.

Then elyra[kfp] would include the KFP sdk, elyra[kfp-tekon] would include elyra[kfp] + kfp-tekton and elyra[airflow] would include this dummy airflow placeholder as its dependency. It could even have a dependency on GitHub, if we needed that (and elyra[gitlab] could include elyra[airflow] + gitlab if we wanted that package tied to airflow.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree. There may be other reasons a package (like github) might be installed. For example, once the community provides a component catalog connector that relies on the package.

except ImportError:
# Github package is not installed, we use GitHub as our default DAG repository
Github = None

from elyra.metadata.schema import SchemasProvider
from elyra.metadata.schemaspaces import CodeSnippets
from elyra.metadata.schemaspaces import ComponentCatalogs
Expand All @@ -51,7 +65,7 @@ class ElyraSchemasProvider(SchemasProvider, metaclass=ABCMeta):
schema_json = json.load(f)
local_schemas.append(schema_json)

def __init__(self):
def __init__(self, *args, **kwargs):
self.log = log.get_logger()
# get set of registered runtimes
self._runtime_processor_names = set()
Expand Down Expand Up @@ -82,17 +96,17 @@ def get_schemas(self) -> List[Dict]:
schemas = self.get_local_schemas_by_schemaspace(Runtimes.RUNTIMES_SCHEMASPACE_ID)
for schema in schemas:
if schema["name"] in self._runtime_processor_names:
runtime_schemas.append(schema)
if schema["name"] == "kfp":
if schema["name"] == "kfp" and compiler:
kfp_schema_present = True
elif schema["name"] == "airflow":
runtime_schemas.append(schema)
elif schema["name"] == "airflow" and Github:
airflow_schema_present = True
runtime_schemas.append(schema)
else:
self.log.error(
f"No entrypoint with name '{schema['name']}' was found in group "
f"'elyra.pipeline.processor' to match the schema with the same name. Skipping..."
)

if kfp_schema_present: # Update the kfp engine enum to reflect current packages...
# If TektonClient package is missing, navigate to the engine property
# and remove 'tekton' entry if present and return updated result.
Expand Down
32 changes: 19 additions & 13 deletions elyra/pipeline/processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
from urllib3.exceptions import MaxRetryError

from elyra.metadata.manager import MetadataManager
from elyra.metadata.schema import SchemaManager
from elyra.metadata.schemaspaces import Runtimes
from elyra.pipeline.component import Component
from elyra.pipeline.component_catalog import ComponentCache
from elyra.pipeline.pipeline import GenericOperation
Expand All @@ -57,19 +59,23 @@ def __init__(self, **kwargs):
self.root_dir = get_expanded_path(kwargs.get("root_dir"))
# Register all known processors based on entrypoint configuration
for processor in entrypoints.get_group_all("elyra.pipeline.processors"):
try:
# instantiate an actual instance of the processor
processor_instance = processor.load()(self.root_dir, parent=kwargs.get("parent")) # Load an instance
self.log.info(
f"Registering {processor.name} processor " f'"{processor.module_name}.{processor.object_name}"...'
)
self.add_processor(processor_instance)
except Exception as err:
# log and ignore initialization errors
self.log.error(
f"Error registering {processor.name} processor "
f'"{processor.module_name}.{processor.object_name}" - {err}'
)
if processor.name in SchemaManager.instance().get_schemasproviders(Runtimes.RUNTIMES_SCHEMASPACE_ID):
try:
# instantiate an actual instance of the processor
processor_instance = processor.load()(
self.root_dir, parent=kwargs.get("parent")
) # Load an instance
self.log.info(
f"Registering {processor.name} processor "
f'"{processor.module_name}.{processor.object_name}"...'
)
self.add_processor(processor_instance)
except Exception as err:
# log and ignore initialization errors
self.log.error(
f"Error registering {processor.name} processor "
f'"{processor.module_name}.{processor.object_name}" - {err}'
)

def add_processor(self, processor):
self.log.debug(f"Registering {processor.type.value} runtime processor '{processor.name}'")
Expand Down
3 changes: 3 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
[build-system]
requires=["jupyter_packaging>=0.9,<2"]

[tool.black]
# max line length; same as 'max-line-length' in setup.cfg
line-length = 120
Expand Down
140 changes: 138 additions & 2 deletions setup.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,148 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
[metadata]
name=elyra
version=attr: elyra._version.__version__
url=https://github.com/elyra-ai/elyra
description=Elyra provides AI Centric extensions to JupyterLab
long_description=Elyra is a set of AI centric extensions to JupyterLab. It aims to help data scientists,
machine learning engineers and AI developer’s through the model development life cycle complexities.
author=Elyra Maintainers
license=Apache License Version 2.0
platforms = Linux, Mac OS X, Windows
classifiers=
License :: OSI Approved :: Apache Software License
Operating System :: OS Independent
Topic :: Scientific/Engineering
Topic :: Scientific/Engineering :: Artificial Intelligence
Topic :: Software Development
Topic :: Software Development :: Libraries
Topic :: Software Development :: Libraries :: Python Modules
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10

[bdist_wheel]
universal=0

[metadata]
description_file=README.md
[options]
zip_safe = False
include_package_data = True
packages = find:
python_requires = >=3.7
install_requires =
autopep8>=1.5.0
black<=21.12b0 # Cap due to psf/black#2846
click>=8 # elyra-ai/elyra#2579
colorama
deprecation
entrypoints>=0.3
jinja2>=3
jsonschema>=3.2.0
jupyter_core>=4.6.0
jupyter_client>=6.1.7
jupyter-packaging>=0.10
jupyter_server>=1.7.0
jupyterlab>=3.3.0
jupyterlab-git~=0.32 # Avoid breaking 1.x changes
jupyterlab-lsp>=3.8.0
jupyter-resource-usage>=0.5.1
MarkupSafe>=2.1
minio>=7.0.0
nbclient>=0.5.1
nbconvert>=6.4.5
nbdime~=3.1 # Cap from jupyterlab-git
nbformat>=5.1.2
networkx>=2.5.1
papermill>=2.3.4
python-lsp-server[all]>=1.1.0
pyyaml>=5.3.1
requests>=2.25.1
rfc3986-validator>=0.1.1
tornado>=6.1.0
typing-extensions>=3.10
traitlets>=4.3.2
urllib3>=1.26.5
watchdog>=2.1.3
websocket-client
yaspin

[options.extras_require]
airflow =
pygithub
airflow-examples =
%(airflow)s
elyra-examples-airflow-catalog
gitlab =
python-gitlab
kfp =
jsonschema>=3.2.0,<4.0
kfp>=1.7.0,<2.0,!=1.7.2
pyyaml>=5.3.1,<6.0
typing-extensions>=3.10,<4
kfp-examples =
%(kfp)s
elyra-examples-kfp-catalog
kfp-tekton =
kfp-tekton ~=1.2.0 # See elyra-ai/elyra/pull/2034 for fix pack pinning
test =
elyra-examples-airflow-catalog
elyra-examples-kfp-catalog
git-python
importlib-resources
pytest>=5.4.1
pytest-console-scripts
pytest-tornasync
pytest_virtualenv
requests-mock
requests-unixsocket
all =
%(airflow)s
%(airflow-examples)s
%(gitlab)s
%(kfp)s
%(kfp-examples)s
%(kfp-tekton)s
%(test)s

[options.entry_points]
console_scripts =
elyra-metadata = elyra.metadata.metadata_app:MetadataApp.main
elyra-pipeline = elyra.cli.pipeline_app:pipeline
jupyter-elyra = elyra.elyra_app:launch_instance
metadata.schemaspaces =
runtimes = elyra.metadata.schemaspaces:Runtimes
runtimes-images = elyra.metadata.schemaspaces:RuntimeImages
code-snippets = elyra.metadata.schemaspaces:CodeSnippets
component-catalogs = elyra.metadata.schemaspaces:ComponentCatalogs
metadata-tests = elyra.tests.metadata.test_utils:MetadataTestSchemaspace
metadata.schemas_providers =
runtimes = elyra.metadata.schemasproviders:RuntimesSchemas
runtimes-images = elyra.metadata.schemasproviders:RuntimeImagesSchemas
code-snippets = elyra.metadata.schemasproviders:CodeSnippetsSchemas
component-catalogs = elyra.metadata.schemasproviders:ComponentCatalogsSchemas
airflow-provider-package-catalog-schema = elyra.pipeline.airflow.provider_package_catalog_connector.airflow_provider_package_schema_provider:AirflowProviderPackageSchemasProvider [airflow]
airflow-package-catalog-schema = elyra.pipeline.airflow.package_catalog_connector.airflow_package_schema_provider:AirflowPackageSchemasProvider [airflow]
metadata-tests = elyra.tests.metadata.test_utils:MetadataTestSchemasProvider
elyra.pipeline.processors =
local = elyra.pipeline.local.processor_local:LocalPipelineProcessor
airflow = elyra.pipeline.airflow.processor_airflow:AirflowPipelineProcessor [airflow]
kfp = elyra.pipeline.kfp.processor_kfp:KfpPipelineProcessor [kfp, kfp_tekton]
elyra.component.catalog_types =
url-catalog = elyra.pipeline.catalog_connector:UrlComponentCatalogConnector
local-file-catalog = elyra.pipeline.catalog_connector:FilesystemComponentCatalogConnector
local-directory-catalog = elyra.pipeline.catalog_connector:DirectoryComponentCatalogConnector
airflow-provider-package-catalog = elyra.pipeline.airflow.provider_package_catalog_connector.airflow_provider_package_catalog_connector:AirflowProviderPackageCatalogConnector [airflow]
airflow-package-catalog = elyra.pipeline.airflow.package_catalog_connector.airflow_package_catalog_connector:AirflowPackageCatalogConnector [airflow]
papermill.engine =
ElyraEngine = elyra.pipeline.elyra_engine:ElyraEngine

[options.packages.find]
exclude =
docs*
tests*

[flake8]
application-import-names = elyra, test
Expand Down
Loading