Merge branch 'main' into oauth2_scope_config

.github/workflows/ci.yml

@@ -8,21 +8,19 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: [ "3.9", "3.10", "3.11" ]
         postgres-version: [12, 15]
         os: [ubuntu-20.04, ubuntu-22.04]
         include:
-          - python-version: "3.8"
-            toxenv: py38
           - python-version: "3.9"
             toxenv: py39
           - python-version: "3.10"
             toxenv: py310
+          - python-version: "3.11"
+            toxenv: py311
         exclude:
           - os: ubuntu-20.04  # python 3.10 isn't working on 20.04 due to missing "libldap-2.5.so.0"
             python-version: "3.10"
-          - os: ubuntu-22.04  # python 3.8 isn't working on 22.04 due to missing "libldap_r-2.4.so.2"
-            python-version: "3.8"
           - os: ubuntu-22.04  # python 3.9 isn't working on 22.04 due to missing "libldap_r-2.4.so.2"
             python-version: "3.9"
       fail-fast: false # run all matrix jobs even if one fails, so we know if the problem is version-specific
@@ -56,7 +54,7 @@ jobs:
       FORCE_COLOR: 1
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -71,7 +69,7 @@ jobs:
         python -m venv ~/env
         source ~/env/bin/activate
         python -m pip install --upgrade pip
-        python -m pip install --upgrade "setuptools<=v65.7.0" # https://github.com/Netflix/lemur/issues/4297
+        python -m pip install --upgrade setuptools
         pip install mypy
         pip install bandit
         sudo apt-get update

.github/workflows/codeql.yml

@@ -13,10 +13,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ "master" ]
+    branches: [ "main" ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ "master" ]
+    branches: [ "main" ]
   schedule:
     - cron: '15 16 * * 2'
 
@@ -41,7 +41,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Install prerequisites for python-ldap. See: https://www.python-ldap.org/en/python-ldap-3.3.0/installing.html#build-prerequisites
     - name: Install python-ldap prerequisites

.github/workflows/dependabot-auto-merge.yml

@@ -7,7 +7,7 @@ jobs:
   auto-merge:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ahmadnassri/action-dependabot-auto-merge@v2
         with:
           target: minor

.github/workflows/lemur-publish-release-pypi.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
       uses: actions/setup-python@v4
       with:

.readthedocs.yml

@@ -17,10 +17,9 @@ formats: all
 build:
    os: ubuntu-22.04
    tools:
-      python: "3.10"
+      python: "3.11"
 python:
    install:
    - requirements: requirements-docs.txt
    - method: setuptools
      path: .
-   system_packages: true

CHANGELOG.rst

@@ -3,8 +3,16 @@ Changelog
 
 Unreleased
 ~~~~~~~~~~~~~~~~~~~~
+Add NTLM auth support for ADCS issuer.
+Added password complexity requirements:
+- At least 12 characters (required for your Muhlenberg password)—the more characters, the better
+- A mixture of both uppercase and lowercase letters
+- A mixture of letters and numbers
+- Inclusion of at least one special character, e.g., ! @ # ? ]
+If you don't want password complexity requirements, you can set CHECK_PASSWORD_STRENGTH to False.
 Added ability to limit authority creation to admins only using config option `ADMIN_ONLY_AUTHORITY_CREATION`.
-
+User passwords can now be updated by admins with the update user endpoint.
+Route53 find_zone_dns now selects the maximum suffix match for zone id (previously we selected the first match).
 
 1.5.0 - `2023-07-05`
 ~~~~~~~~~~~~~~~~~~~~

Dockerfile

@@ -1,9 +1,9 @@
-FROM python:3.7
+FROM python:3.8
 RUN apt-get update
 RUN apt-get install -y make software-properties-common curl
-RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
+RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -
 RUN apt-get update
-RUN apt-get install -y npm libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+RUN apt-get install -y nodejs libldap2-dev libsasl2-dev libldap2-dev libssl-dev
 RUN pip install pip==20.0.2
 RUN pip install -U setuptools
 RUN pip install coveralls bandit

README.rst

@@ -11,8 +11,8 @@ Lemur
 
 .. image:: https://img.shields.io/badge/NetflixOSS-active-brightgreen.svg
 
-.. image:: https://coveralls.io/repos/github/Netflix/lemur/badge.svg?branch=master
-    :target: https://coveralls.io/github/Netflix/lemur?branch=master
+.. image:: https://coveralls.io/repos/github/Netflix/lemur/badge.svg?branch=main
+    :target: https://coveralls.io/github/Netflix/lemur?branch=main
 
 
 

docs/administration.rst

@@ -35,6 +35,37 @@ Basic Configuration
 
         LOG_UPGRADE_FILE = "/logs/lemur/db_upgrade.log"
 
+.. data:: LOG_REQUEST_HEADERS
+    :noindex:
+
+    ::
+        Defaults to False (off).  This adds logging similar to a webserver, where each request made to the API is logged.
+        Useful for tracing where requests are being made from, or for auditing purposes.
+
+        LOG_REQUEST_HEADERS = True
+
+.. data:: LOG_SANITIZE_REQUEST_HEADERS
+    :noindex:
+
+    ::
+        Defaults to True (on).  This sanitizes the requests logging to remove the query parameters,
+        as those parameters often contain sensitivity information.
+
+        LOG_SANITIZE_REQUEST_HEADERS = True
+
+    .. warning::
+        This should never be used in a production environment as it exposes sensitivite information.
+
+.. data:: LOG_REQUEST_HEADERS_SKIP_ENDPOINT
+    :noindex:
+
+    ::
+        Defaults to the metrics and healthcheck endpoints.  Some endpoints are not useful to log and can generate a lot of noise.
+        If an endpoint is listed here, it will be skipped and not logged.  It is only recommended to add endpoints that are purely
+        informational or only used internally.
+
+        LOG_REQUEST_HEADERS_SKIP_ENDPOINT = ["/metrics", "/healthcheck"]
+
 .. data:: DEBUG
     :noindex:
 
@@ -247,6 +278,20 @@ Basic Configuration
        Specifies the AWS partition that Lemur should use. Valid values are 'aws', 'aws-us-gov', and 'aws-cn'. Defaults to 'aws'.
        If Lemur is deployed in and managing endpoints AWS GovCloud, for example, you must set this to `aws-us-gov`.
 
+
+.. data:: LEMUR_STRICT_ROLE_ENFORCEMENT
+    :noindex:
+
+        When set to True, this property enforces the default Lemur role functionality. The default Lemur roles are
+        ``admin``, ``operator``, and ``read-only``. Users will be required to have a default role assigned
+        upon creation. The ``operator`` and ``read-only`` roles are strictly enforced. Users assigned the ``read-only``
+        role will not be able to create/update resources.
+
+    ::
+
+        LEMUR_STRICT_ROLE_ENFORCEMENT = False
+
+
 .. data:: SENTRY_DSN
     :noindex:
 
@@ -722,6 +767,182 @@ Authentication Options
 ----------------------
 Lemur currently supports Basic Authentication, LDAP Authentication, Ping OAuth2, and Google out of the box. Additional flows can be added relatively easily.
 
+IDP Configuration Options
+-------------------------
+Lemur historically was hard coded for supporting creating or adding groups from the `googleGroups` key. This was great if you used Google, but a limitation
+if you used another IDP. These settings allow you to customize your IDP and do one of two things:
+* Uses a fixed mapping in our config, if the user has that group we give a specified Lemur role.
+* Dynamically takes all the groups assigned to a user (such as `profile['roles']` or `profile['googleGroups']`), creates a Lemur group if it doesn't already exist, then assigns it to the user.
+
+Group to role mapping can be either very static with a hard mapping (`IDP_ROLES_MAPPING`) or entirely dynamic from your designated IDP groups (`IDP_GROUPS_KEYS`).
+Using both methods at once is supported.
+
+Here is an example IDP configuration stanza you can add to your config. Adjust to suit your environment of course.
+
+.. code-block:: python
+
+        IDP_ROLES_MAPPING = {"security": "admin", "engineering": "operator", "jane_from_accounting": "read-only"}
+        IDP_GROUPS_KEYS = ["googleGroups", "roles"]
+        IDP_ASSIGN_ROLES_FROM_USER_GROUPS = True
+        IDP_CREATE_ROLES_FROM_USER_GROUPS = True
+        IDP_CREATE_PER_USER_ROLE = True
+        IDP_PROTECT_BUILTINS = True
+
+.. data:: IDP_ROLES_MAPPING
+    :noindex:
+
+        Defaults to `{}`.
+
+        This is used to create a fixed user mapping from an IDP group to a Lemur group.  This is very useful if you have do very static assigments using a set number of groups.
+        The dynamic group assignment will not assign built-in groups (`admin`, `operator`, or `read-only`) by default, using `IDP_ROLES_MAPPING` is the recommended way to assign
+        these groups.
+
+        Custom created Lemur groups may be assigned through `IDP_ROLES_MAPPING`, but they must be created before attempting to assign them, this will not automatically create any groups.
+
+        The dictionary is defined with the IDP group as the dictionary key, and the matching Lemur group as the value. (e.g. `{"idp_group": "lemur_group"`)
+
+        ::
+
+            IDP_ROLES_MAPPING = {"security": "admin", "engineering": "operator", "jane_from_accounting": "read-only"}
+
+.. data:: IDP_GROUPS_KEYS
+    :noindex:
+
+        Defaults to `["googleGroups"]` as this is historically what was hard coded in Lemur.
+
+        Provide a list of dictionary keys used by IDP(s) to return user groups.  In code, you can look at the dictionary object provided by your IDP and find the dictionary key that contains
+        the groups you would like to add.  You can also check in Lemur code and look at the `profile` dictionary object for the user.
+
+        You can chain as many fields as you desire, this could be useful if pulling from multiple IDPs or if an IDP provides two different types of groups.
+
+        .. note::
+            Here is a list of key names that may be useful. Double check that these keys match your environment as your results may differ.
+
+            GCP: `googleGroups`
+            LDAP: `roles`
+
+        .. warning::
+            Any keys provided here will be used additively, so care must be taken as unintentional group assignments may be made if using multiple IDPs.
+
+            It is highly recommended to use `IDP_ROLES_PREFIX` and/or `IDP_ROLES_SUFFIX` to create a naming schema to avoid importing all groups assigned in your IDP.
+
+            The groups that are built into Lemur (`admin`, `operator`, `read-only`) will not be assigned by default. See: `IDP_PROTECT_BUILTINS`
+
+        ::
+
+            IDP_GROUPS_KEYS = ["googleGroups"]
+
+.. data:: IDP_ROLES_PREFIX
+    :noindex:
+
+        Defaults to `""`. (No filtering, all groups will be included)
+
+        This is used to create a naming schema/convention to provide a method for only importing a certain set of groups.  A simple string comparison is used
+        to see if the group name starts with the prefix provided here.  If it matches the group will be used; if it doesn't it will be skipped.
+
+        ::
+
+            IDP_ROLES_PREFIX = "LEMUR-"
+
+.. data:: IDP_ROLES_SUFFIX
+    :noindex:
+
+        Defaults to `""`. (No filtering, all groups will be included)
+
+        This is used to create a naming schema/convention to provide a method for only importing a certain set of groups.  A simple string comparison is used
+        to see if the group name ends with the suffix provided here.  If it matches the group will be used; if it doesn't it will be skipped.
+
+        ::
+
+            IDP_ROLES_SUFFIX = "_LEMUR"
+
+.. data:: IDP_ROLES_DESCRIPTION
+    :noindex:
+
+        Defaults to `Identity provider role from '{idp_groups_key}' (generated by Lemur)`.
+
+        Allows for a custom group description for automatically created groups.
+
+        .. note::
+            While the default description will automatically inject the IDP group key into the description, a custom version will not.
+            If you are using multiple group fields or IDPs, you may want to leave this at the default.
+
+        ::
+
+            IDP_ROLES_DESCRIPTION = "Automatically generated role from my IDP"
+
+.. data:: IDP_ASSIGN_ROLES_FROM_USER_GROUPS
+    :noindex:
+
+        Defaults to `True`.
+
+        Enables automatic assignment of groups found in the IDP key to the user.  This is used to dynamically assign groups based on your IDP groups assignments.
+
+        .. note::
+            This does not create groups if they are not found, for that functionality see `IDP_CREATE_ROLES_FROM_USER_GROUPS`.
+
+        .. warning::
+            While automatic group assignment can be very useful, this can potentially assign groups unexpectedly as once this is enabled group assignment is done from the IDP.
+            If you want more strict or controlled group assignments, it is recommended to use `IDP_ROLES_MAPPING`.
+
+        ::
+
+            IDP_ASSIGN_ROLES_FROM_USER_GROUPS = True
+
+.. data:: IDP_CREATE_ROLES_FROM_USER_GROUPS
+    :noindex:
+
+        Defaults to `True`.
+
+        Enables automatic creation of groups found in the IDP key to the user.  This is used to dynamically create groups based on your IDP groups assignments.
+
+        .. note::
+            This does not assign groups to a user, for that functionality see `IDP_ASSIGN_ROLES_FROM_USER_GROUPS`.
+
+        .. warning::
+            While automatic group creation can be very useful, this can potentially create groups unexpectedly as once this is enabled group assignment is done from the IDP.
+            If you want more strict or controlled group assignments, it is recommended to use `IDP_ROLES_MAPPING`.
+
+        ::
+
+            IDP_CREATE_ROLES_FROM_USER_GROUPS = True
+
+
+.. data:: IDP_CREATE_PER_USER_ROLE
+    :noindex:
+
+        Defaults to `True`.
+
+        Creates a group for each user.  While this is very noisy, this can be useful to assign a cert to a singular user.
+
+        .. note::
+            Organizations with very large numbers of users may want to condsider disabling this, due to the potential load.
+
+        ::
+
+            IDP_CREATE_PER_USER_ROLE = True
+
+.. data:: IDP_PROTECT_BUILTINS
+    :noindex:
+
+        Defaults to `True`.
+
+        Prevents dynamic assignment of the built in groups (`admin`, `operator`, `read-only`) to users. If a group provided by the IDP key has these values,
+        it is skipped.  The built in groups are protected by default as accidential adding assignments is very easy to do.  For example, an LDAP administrator
+        adds a group named `admin` and now all of your LDAP admins are also Lemur admins.
+
+        If you want to assign built in groups, it is recommended to use `IDP_ROLES_MAPPING`.
+
+        .. note::
+            Historically Lemur allowed for dynamic assignment of the built-in groups.
+
+        .. warning::
+            Disabling this protection is not recommended.
+
+        ::
+
+            IDP_PROTECT_BUILTINS = True
+
 LDAP Options
 ~~~~~~~~~~~~
 
@@ -1823,7 +2044,7 @@ To get the latest code from github run
 
 .. note::
     It's important to grab the latest release by specifying the release tag. This tags denote stable versions of Lemur.
-    If you want to try the bleeding edge version of Lemur you can by using the master branch.
+    If you want to try the bleeding edge version of Lemur you can by using the main branch.
 
 
 After you have the latest version of the Lemur code base you must run any needed database migrations. To run migrations

docs/developer/index.rst

@@ -227,7 +227,7 @@ Uncertain about how to write tests? Take a look at some existing tests that are
 
 You can see a list of open pull requests (pending changes) by visiting https://github.com/netflix/lemur/pulls
 
-Pull requests should be against **master** and pass all TravisCI checks
+Pull requests should be against **main** and pass all TravisCI checks
 
 
 Writing a Plugin