Improves installing from sources pages for all components (#18251)

* Shorter menu sections for installation page * Added "installing from sources" for Helm Chart * Added Providers summary page for all provider packages * Added scripts to verify PyPI packages with gpg/sha
apache · Sep 15, 2021 · 67fddbf · 67fddbf
1 parent 292751c
commit 67fddbf
Show file tree

Hide file tree

Showing 8 changed files with 389 additions and 31 deletions.
diff --git a/docs/apache-airflow-providers/index.rst b/docs/apache-airflow-providers/index.rst
@@ -407,14 +407,20 @@ provider package will keep its own dependencies, and while we expect those provi
 backwards-compatible, particular versions of particular providers might introduce dependencies on
 specific Airflow versions.
 
-Contents
---------
 
 .. toctree::
+    :hidden:
     :maxdepth: 2
 
     Providers <self>
     Packages <packages-ref>
     Operators and hooks <operators-and-hooks-ref/index>
     Core Extensions <core-extensions/index>
     Update community providers <howto/create-update-providers>
+
+.. toctree::
+    :maxdepth: 2
+    :hidden:
+    :caption: Resources
+
+    Installing from sources <installing-from-sources>
diff --git a/docs/apache-airflow-providers/installing-from-sources.rst b/docs/apache-airflow-providers/installing-from-sources.rst
@@ -0,0 +1,117 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+Installing Providers from Sources
+---------------------------------
+
+.. contents:: :local:
+
+
+Released packages
+'''''''''''''''''
+
+.. jinja:: official_download_page
+
+    .. raw:: html
+
+        <ul style="column-count: 2;">
+        {% for provider in all_providers %}
+            <li><a href="/docs/{{ provider['package-name'] }}/latest/installing-providers-from-sources.html"><code>{{ provider.name }}</code></a></li>
+        {% endfor %}
+         </ul>
+
+
+Release integrity
+'''''''''''''''''
+
+`PGP signatures KEYS <https://downloads.apache.org/airflow/KEYS>`_
+
+It is essential that you verify the integrity of the downloaded files using the PGP or SHA signatures.
+The PGP signatures can be verified using GPG or PGP. Please download the KEYS as well as the asc
+signature files for relevant distribution. It is recommended to get these files from the
+main distribution directory and not from the mirrors.
+
+.. code-block:: bash
+
+    gpg -i KEYS
+
+or
+
+.. code-block:: bash
+
+    pgpk -a KEYS
+
+or
+
+.. code-block:: bash
+
+    pgp -ka KEYS
+
+To verify the binaries/sources you can download the relevant asc files for it from main
+distribution directory and follow the below guide.
+
+.. code-block:: bash
+
+    gpg --verify apache-airflow-providers-********.asc apache-airflow-providers-*********
+
+or
+
+.. code-block:: bash
+
+    pgpv apache-airflow-providers-********.asc
+
+or
+
+.. code-block:: bash
+
+    pgp apache-airflow-providers-********.asc
+
+Example:
+
+.. code-block:: console
+    :substitutions:
+
+    $ gpg --verify apache-airflow-providers-airbyte-1.0.0-source.tar.gz.asc apache-airflow-providers-airbyte-1.0.0-source.tar.gz
+      gpg: Signature made Sat 11 Sep 12:49:54 2021 BST
+      gpg:                using RSA key CDE15C6E4D3A8EC4ECF4BA4B6674E08AD7DE406F
+      gpg:                issuer "kaxilnaik@apache.org"
+      gpg: Good signature from "Kaxil Naik <kaxilnaik@apache.org>" [unknown]
+      gpg:                 aka "Kaxil Naik <kaxilnaik@gmail.com>" [unknown]
+      gpg: WARNING: The key's User ID is not certified with a trusted signature!
+      gpg:          There is no indication that the signature belongs to the owner.
+      Primary key fingerprint: CDE1 5C6E 4D3A 8EC4 ECF4  BA4B 6674 E08A D7DE 406F
+
+The "Good signature from ..." is indication that the signatures are correct.
+Do not worry about the "not certified with a trusted signature" warning. Most of the certificates used
+by release managers are self signed, that's why you get this warning. By importing the server in the
+previous step and importing it via ID from ``KEYS`` page, you know that this is a valid Key already.
+
+For SHA512 sum check, download the relevant ``sha512`` and run the following:
+
+.. code-block:: bash
+
+    shasum -a 512 apache-airflow-providers-********  | diff - apache-airflow-providers-********.sha512
+
+The ``SHASUM`` of the file should match the one provided in ``.sha512`` file.
+
+Example:
+
+.. code-block:: bash
+    :substitutions:
+
+    shasum -a 512 apache-airflow-providers-airbyte-1.0.0-source.tar.gz  | diff - apache-airflow-providers-airbyte-1.0.0-source.tar.gz.sha512
diff --git a/docs/apache-airflow/index.rst b/docs/apache-airflow/index.rst
@@ -78,7 +78,6 @@ unit of work and continuity.
     license
     start/index
     installation
-    installing-from-sources
     upgrading-to-2
     upgrade-check
     tutorial
@@ -118,3 +117,9 @@ unit of work and continuity.
     Configurations <configurations-ref>
     Extra packages <extra-packages-ref>
     Database Migrations <migrations-ref>
+
+.. toctree::
+    :hidden:
+    :caption: Resources
+
+    Installing from sources <installing-from-sources>
diff --git a/docs/apache-airflow/installing-from-sources.rst b/docs/apache-airflow/installing-from-sources.rst
@@ -15,38 +15,45 @@
     specific language governing permissions and limitations
     under the License.
 
-
 Installing from Sources
 -----------------------
 
 .. contents:: :local:
 
 
-Installing Airflow from released sources and packages
-'''''''''''''''''''''''''''''''''''''''''''''''''''''
+Released packages
+'''''''''''''''''
+
+.. jinja:: official_download_page
+
+    This page describes downloading and verifying ``Apache Airflow`` version
+    ``{{ airflow_version }}`` using officially released packages.
+    You can also install ``Apache Airflow`` - as most Python packages - via
+    `PyPI <https://pypi.org/project/apache-airflow/{{ airflow_version }}>`__ .
+    You can choose different version of Airflow by selecting different version from the drop-down at
+    the top-left of the page.
 
-You can also install Airflow using the official sources and packages. Those sources and packages
-released are the "official" sources of installation that you can use if you want to verify the
-origin of the packages and want to verify checksums and signatures of the packages.
+The ``source``, ``sdist`` and ``whl`` packages released are the "official" sources of installation that you
+can use if you want to verify the origin of the packages and want to verify checksums and signatures of
+the packages. The packages are available via the
+`Official Apache Software Foundations Mirrors <http://ws.apache.org/mirrors.cgi>`_
 
-The packages are available at the
-`Official Apache Software Foundations Downloads page <https://downloads.apache.org/airflow/>`_
 
 The |version| downloads are available at:
 
 .. jinja:: official_download_page
 
-    * `Sdist package <{{ closer_lua_url }}/apache-airflow-{{ airflow_version }}.tar.gz>`_ (`asc <{{ base_url }}/apache-airflow-{{ airflow_version }}.tar.gz.asc>`__, `sha512 <{{ base_url }}/apache-airflow-{{ airflow_version }}.tar.gz.sha512>`__)
-    * `Wheel package <{{ closer_lua_url }}/apache_airflow-{{ airflow_version }}-py3-none-any.whl>`_ (`asc <{{ base_url }}/apache_airflow-{{ airflow_version }}-py3-none-any.whl.asc>`__, `sha512 <{{ base_url }}/apache_airflow-{{ airflow_version }}-py3-none-any.whl.sha512>`__)
-    * `Sources <{{ closer_lua_url }}/apache-airflow-{{ airflow_version }}-source.tar.gz>`_ (`asc <{{ base_url }}/apache_airflow-{{ airflow_version }}-source.tar.gz.asc>`__, `sha512 <{{ base_url }}/apache-airflow-{{ airflow_version }}-source.tar.gz.sha512>`__)
+    * `Sources package <{{ closer_lua_url }}/apache-airflow-{{ airflow_version }}-source.tar.gz>`__ (`asc <{{ base_url }}/apache_airflow-{{ airflow_version }}-source.tar.gz.asc>`__, `sha512 <{{ base_url }}/apache-airflow-{{ airflow_version }}-source.tar.gz.sha512>`__)
+    * `Sdist package <{{ closer_lua_url }}/apache-airflow-{{ airflow_version }}.tar.gz>`__ (`asc <{{ base_url }}/apache-airflow-{{ airflow_version }}.tar.gz.asc>`__, `sha512 <{{ base_url }}/apache-airflow-{{ airflow_version }}.tar.gz.sha512>`__)
+    * `Whl package <{{ closer_lua_url }}/apache_airflow-{{ airflow_version }}-py3-none-any.whl>`__ (`asc <{{ base_url }}/apache_airflow-{{ airflow_version }}-py3-none-any.whl.asc>`__, `sha512 <{{ base_url }}/apache_airflow-{{ airflow_version }}-py3-none-any.whl.sha512>`__)
 
 If you want to install from the source code, you can download from the sources link above, it will contain
 a ``INSTALL`` file containing details on how you can build and install Airflow.
 
-Release integrity & Verification of releases
-''''''''''''''''''''''''''''''''''''''''''''
+Release integrity
+'''''''''''''''''
 
-`PGP signatures KEYS <https://downloads.apache.org/airflow/KEYS>`_
+`PGP signatures KEYS <https://downloads.apache.org/airflow/KEYS>`__
 
 It is essential that you verify the integrity of the downloaded files using the PGP or SHA signatures.
 The PGP signatures can be verified using GPG or PGP. Please download the KEYS as well as the asc
@@ -122,3 +129,31 @@ Example:
     :substitutions:
 
     shasum -a 512 apache-airflow-|version|-source.tar.gz  | diff - apache-airflow-|version|-source.tar.gz.sha512
+
+
+Verifying PyPI releases
+'''''''''''''''''''''''
+
+You can verify the Airflow ``.whl`` packages from PyPI by locally downloading the package and signature
+and SHA sum files with the script below:
+
+
+.. jinja:: official_download_page
+
+    .. code-block:: bash
+
+        #!/bin/bash
+        AIRFLOW_VERSION="{{ airflow_version }}"
+        airflow_download_dir="$(mktemp -d)"
+        pip download --no-deps "apache-airflow==${AIRFLOW_VERSION}" --dest "${airflow_download_dir}"
+        curl "https://downloads.apache.org/airflow/${AIRFLOW_VERSION}/apache_airflow-${AIRFLOW_VERSION}-py3-none-any.whl.asc" \
+            -L -o "${airflow_download_dir}/apache_airflow-${AIRFLOW_VERSION}-py3-none-any.whl.asc"
+        curl "https://downloads.apache.org/airflow/${AIRFLOW_VERSION}/apache_airflow-${AIRFLOW_VERSION}-py3-none-any.whl.sha512" \
+            -L -o "${airflow_download_dir}/apache_airflow-${AIRFLOW_VERSION}-py3-none-any.whl.sha512"
+        echo
+        echo "Please verify files downloaded to ${airflow_download_dir}"
+        ls -la "${airflow_download_dir}"
+        echo
+
+Once you verify the files following the instructions from previous chapter you can remove the temporary
+folder created.
diff --git a/docs/conf.py b/docs/conf.py
@@ -75,6 +75,12 @@
         raise Exception(f"Could not find provider.yaml file for package: {PACKAGE_NAME}")
     PACKAGE_DIR = CURRENT_PROVIDER['package-dir']
     PACKAGE_VERSION = CURRENT_PROVIDER['versions'][0]
+elif PACKAGE_NAME == 'apache-airflow-providers':
+    from provider_yaml_utils import load_package_data
+
+    PACKAGE_DIR = os.path.join(ROOT_DIR, 'airflow', 'providers')
+    PACKAGE_VERSION = 'devel'
+    ALL_PROVIDER_YAMLS = load_package_data()
 elif PACKAGE_NAME == 'helm-chart':
     PACKAGE_DIR = os.path.join(ROOT_DIR, 'chart')
     CHART_YAML_FILE = os.path.join(PACKAGE_DIR, 'Chart.yaml')
@@ -155,6 +161,7 @@
 if PACKAGE_NAME == "apache-airflow-providers":
     extensions.extend(
         [
+            'sphinxcontrib.jinja',
             'operators_and_hooks_ref',
             'providers_packages_ref',
         ]
@@ -407,6 +414,12 @@ def _load_config():
             'package_version': PACKAGE_VERSION,
         },
     }
+elif PACKAGE_NAME == 'apache-airflow-providers':
+    jinja_contexts = {
+        'official_download_page': {
+            'all_providers': ALL_PROVIDER_YAMLS,
+        },
+    }
 elif PACKAGE_NAME == 'helm-chart':
 
     def _str_representer(dumper, data):
@@ -486,7 +499,15 @@ def _get_params(root_schema: dict, prefix: str = "", default_section: str = "")
     if sections:
         raise ValueError(f"Found section(s) which were not in `section_order`: {list(sections.keys())}")
 
-    jinja_contexts = {"params_ctx": {"sections": ordered_sections}}
+    jinja_contexts = {
+        "params_ctx": {"sections": ordered_sections},
+        'official_download_page': {
+            'base_url': 'https://downloads.apache.org/airflow/helm-chart',
+            'closer_lua_url': 'https://www.apache.org/dyn/closer.lua/airflow/helm-chart',
+            'package_name': PACKAGE_NAME,
+            'package_version': PACKAGE_VERSION,
+        },
+    }
 
 
 # -- Options for sphinx.ext.autodoc --------------------------------------------

diff --git a/docs/helm-chart/index.rst b/docs/helm-chart/index.rst
@@ -47,6 +47,12 @@ Helm Chart for Apache Airflow
     changelog
     Updating <updating>
 
+.. toctree::
+    :hidden:
+    :caption: Resources
+
+    Installing from sources<installing-helm-chart-from-sources>
+
 
 This chart will bootstrap an `Airflow <https://airflow.apache.org>`__
 deployment on a `Kubernetes <http://kubernetes.io>`__ cluster using the