Introduces separate runtime provider schema

.pre-commit-config.yaml

@@ -500,11 +500,7 @@ repos:
  - https://json-schema.org/draft-07/schema
  language: python
  pass_filenames: true
- files: >
- (?x)
- ^airflow/provider.yaml.schema.json$|
- ^airflow/config_templates/config.yml.schema.json$|
- ^airflow/serialization/schema.json$
+ files: .*\.schema\.json$
  require_serial: true
  additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
  - id: json-schema

MANIFEST.in

@@ -32,7 +32,7 @@ global-exclude __pycache__ *.pyc
 include airflow/alembic.ini
 include airflow/api_connexion/openapi/v1.yaml
 include airflow/git_version
-include airflow/provider.yaml.schema.json
+include airflow/provider_info.schema.json
 include airflow/customized_form_field_behaviours.schema.json
 include airflow/serialization/schema.json
 include airflow/utils/python_virtualenv_script.jinja2

TESTING.rst

@@ -973,7 +973,7 @@ If ``current`` is specified (default), then the current version of Airflow is us
 Otherwise, the released version of Airflow is installed.
 
 The ``-install-airflow-version=<VERSION>`` command make sure that the current (from sources) version of
-Airflow is removed and the released version of Airflow from ``Pypi`` is installed. Note that tests sources
+Airflow is removed and the released version of Airflow from ``PyPI`` is installed. Note that tests sources
 are not removed and they can be used to run tests (unit tests and system tests) against the
 freshly installed version.
 

airflow/deprecated_schemas/provider-2.0.0.yaml.schema.json

@@ -15,7 +15,7 @@
  "type": "string"
  },
  "versions": {
- "description": "List of available versions in Pypi. Sorted descending according to release date.",
+ "description": "List of available versions in PyPI. Sorted descending according to release date.",
  "type": "array",
  "items": {
  "type": "string"

airflow/provider.yaml.schema.json

@@ -15,7 +15,7 @@
  "type": "string"
  },
  "versions": {
- "description": "List of available versions in Pypi. Sorted descending according to release date.",
+ "description": "List of available versions in PyPI. Sorted descending according to release date.",
  "type": "array",
  "items": {
  "type": "string"
@@ -68,7 +68,7 @@
  "maxItems": 1
  }
  },
- "additionalProperties": true,
+ "additionalProperties": false,
  "required": [
  "integration-name",
  "external-doc-url",
@@ -93,7 +93,7 @@
  }
  }
  },
- "additionalProperties": true,
+ "additionalProperties": false,
  "required": [
  "integration-name",
  "python-modules"
@@ -141,7 +141,7 @@
  }
  }
  },
- "additionalProperties": true,
+ "additionalProperties": false,
  "required": [
  "integration-name",
  "python-modules"
@@ -170,7 +170,7 @@
  "description": "List of python modules containing the transfers."
  }
  },
- "additionalProperties": true,
+ "additionalProperties": false,
  "required": [
  "source-integration-name",
  "target-integration-name",
@@ -193,7 +193,7 @@
  }
  }
  },
- "additionalProperties": true,
+ "additionalProperties": false,
  "required": [
  "name",
  "package-name",

airflow/provider_info.schema.json

@@ -0,0 +1,38 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "properties": {
+ "package-name": {
+ "description": "Package name available under which the package is available in the PyPI repository.",
+ "type": "string"
+ },
+ "name": {
+ "description": "Provider name",
+ "type": "string"
+ },
+ "description": {
+ "description": "Information about the package in RST format",
+ "type": "string"
+ },
+ "hook-class-names": {
+ "type": "array",
+ "description": "Hook class names that provide connection types to core",
+ "items": {
+ "type": "string"
+ }
+ },
+ "extra-links": {
+ "type": "array",
+ "description": "Class name that provide extra link functionality",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "name",
+ "package-name",
+ "description",
+ "versions"
+ ]
+}

airflow/providers_manager.py

@@ -39,9 +39,9 @@
 log = logging.getLogger(__name__)
 
 
-def _create_provider_schema_validator():
- """Creates JSON schema validator from the provider.yaml.schema.json"""
- schema = json.loads(importlib_resources.read_text('airflow', 'provider.yaml.schema.json'))
+def _create_provider_info_schema_validator():
+ """Creates JSON schema validator from the provider_info.schema.json"""
+ schema = json.loads(importlib_resources.read_text('airflow', 'provider_info.schema.json'))
  cls = jsonschema.validators.validator_for(schema)
  validator = cls(schema)
  return validator
@@ -106,15 +106,15 @@ def __init__(self):
  # Customizations for javascript fields are kept here
  self._field_behaviours: Dict[str, Dict] = {}
  self._extra_link_class_name_set: Set[str] = set()
- self._provider_schema_validator = _create_provider_schema_validator()
+ self._provider_schema_validator = _create_provider_info_schema_validator()
  self._customized_form_fields_schema_validator = (
  _create_customized_form_field_behaviours_schema_validator()
  )
  self._initialized = False
 
  def initialize_providers_manager(self):
  """Lazy initialization of provider data."""
- # We cannot use @cache here because it does not work during pytests, apparently each test
+ # We cannot use @cache here because it does not work during pytest, apparently each test
  # runs it it's own namespace and ProvidersManager is a different object in each namespace
  # even if it is singleton but @cache on the initialize_providers_manager message still works in the
  # way that it is called only once for one of the objects (at least this is how it looks like
@@ -139,7 +139,10 @@ def _discover_all_providers_from_packages(self) -> None:
  """
  Discovers all providers by scanning packages installed. The list of providers should be returned
  via the 'apache_airflow_provider' entrypoint as a dictionary conforming to the
- 'airflow/provider.yaml.schema.json' schema.
+ 'airflow/provider_info.schema.json' schema. Note that the schema is different at runtime
+ than provider.yaml.schema.json. The development version of provider schema is more strict and changes
+ together with the code. The runtime version is more relaxed (allows for additional properties)
+ and verifies only the subset of fields that are needed at runtime.
  """
  for entry_point, dist in entry_points_with_dist('apache_airflow_provider'):
  package_name = dist.metadata['name']
@@ -194,7 +197,7 @@ def _add_provider_info_from_local_source_files_on_path(self, path) -> None:
  for folder, subdirs, files in os.walk(path, topdown=True):
  for filename in fnmatch.filter(files, "provider.yaml"):
  package_name = "apache-airflow-providers" + folder[len(root_path) :].replace(os.sep, "-")
- # We are skipping discovering snowflake because of snowflake monkeypatching problem
+ # We are skipping discovering snowflake because of snowflake monkey-patching problem
  # This is only for local development - it has no impact for the packaged snowflake provider
  # That should work on its own
  # https://github.com/apache/airflow/issues/12881