feat: create new layer_nlp lambda layer

api/terraform/lambda_layer_genai.tf

@@ -8,55 +8,55 @@
 #         that includes the packages required by OpenAI API and LangChain.
 #------------------------------------------------------------------------------
 locals {
-  layer_slug              = "genai"
-  layer_name              = "layer_${local.layer_slug}"
-  layer_parent_directory  = "${path.module}/python"
-  layer_source_directory  = "${local.layer_parent_directory}/${local.layer_name}"
-  layer_packaging_script  = "${local.layer_source_directory}/create_container.sh"
-  layer_package_folder    = local.layer_slug
-  layer_dist_build_path   = "${path.module}/build/"
-  layer_dist_package_name = "${local.layer_name}_dst.zip"
+  genai_layer_slug              = "genai"
+  genai_layer_name              = "layer_${local.genai_layer_slug}"
+  genai_layer_parent_directory  = "${path.module}/python"
+  genai_layer_source_directory  = "${local.genai_layer_parent_directory}/${local.genai_layer_name}"
+  genai_layer_packaging_script  = "${local.genai_layer_source_directory}/create_container.sh"
+  genai_layer_package_folder    = local.genai_layer_slug
+  genai_layer_dist_build_path   = "${path.module}/build/"
+  genai_layer_dist_package_name = "${local.genai_layer_name}_dst.zip"
 }
 
 ###############################################################################
 # Python package
 # https://alek-cora-glez.medium.com/deploying-aws-lambda-function-with-terraform-custom-dependencies-7874407cd4fc
 ###############################################################################
-resource "null_resource" "package_layer_genai" {
+resource "null_resource" "package_genai_layer" {
   triggers = {
     redeployment = sha1(jsonencode([
       "${path.module}/lambda_layer.tf",
-      file("${local.layer_packaging_script}"),
-      file("${local.layer_source_directory}/Dockerfile"),
-      file("${local.layer_source_directory}/create_container.sh"),
-      file("${local.layer_source_directory}/requirements.txt"),
-      fileexists("${local.layer_source_directory}/${local.layer_dist_package_name}") ? filebase64("${local.layer_source_directory}/${local.layer_dist_package_name}") : null
+      file("${local.genai_layer_packaging_script}"),
+      file("${local.genai_layer_source_directory}/Dockerfile"),
+      file("${local.genai_layer_source_directory}/create_container.sh"),
+      file("${local.genai_layer_source_directory}/requirements.txt"),
+      fileexists("${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}") ? filebase64("${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}") : null
     ]))
   }
 
   provisioner "local-exec" {
     interpreter = ["/bin/bash"]
-    command     = local.layer_packaging_script
+    command     = local.genai_layer_packaging_script
 
     environment = {
-      SOURCE_CODE_PATH = local.layer_source_directory
+      SOURCE_CODE_PATH = local.genai_layer_source_directory
       RUNTIME          = var.lambda_python_runtime
-      CONTAINER_NAME   = local.layer_name
-      PACKAGE_NAME     = local.layer_dist_package_name
+      CONTAINER_NAME   = local.genai_layer_name
+      PACKAGE_NAME     = local.genai_layer_dist_package_name
     }
   }
 }
 
 resource "aws_lambda_layer_version" "genai" {
-  filename                 = "${local.layer_source_directory}/${local.layer_dist_package_name}"
-  source_code_hash         = fileexists("${local.layer_source_directory}/${local.layer_dist_package_name}") ? filebase64sha256("${local.layer_source_directory}/${local.layer_dist_package_name}") : null
-  layer_name               = local.layer_slug
+  filename                 = "${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}"
+  source_code_hash         = fileexists("${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}") ? filebase64sha256("${local.genai_layer_source_directory}/${local.genai_layer_dist_package_name}") : null
+  layer_name               = local.genai_layer_slug
   compatible_architectures = var.compatible_architectures
   compatible_runtimes      = [var.lambda_python_runtime]
   lifecycle {
     create_before_destroy = true
   }
   depends_on = [
-    null_resource.package_layer_genai
+    null_resource.package_genai_layer
   ]
 }

api/terraform/lambda_layer_nlp.tf

@@ -0,0 +1,62 @@
+#------------------------------------------------------------------------------
+# written by: Lawrence McDaniel
+#             https://lawrencemcdaniel.com/
+#
+# date:   sep-2023
+#
+# usage:  implement a Python Lambda layer with a virtual environment
+#         that includes the packages required by OpenAI API and LangChain.
+#------------------------------------------------------------------------------
+locals {
+  nlp_layer_slug              = "nlp"
+  nlp_layer_name              = "layer_${local.nlp_layer_slug}"
+  nlp_layer_parent_directory  = "${path.module}/python"
+  nlp_layer_source_directory  = "${local.nlp_layer_parent_directory}/${local.nlp_layer_name}"
+  nlp_layer_packaging_script  = "${local.nlp_layer_source_directory}/create_container.sh"
+  nlp_layer_package_folder    = local.nlp_layer_slug
+  nlp_layer_dist_build_path   = "${path.module}/build/"
+  nlp_layer_dist_package_name = "${local.nlp_layer_name}_dst.zip"
+}
+
+###############################################################################
+# Python package
+# https://alek-cora-glez.medium.com/deploying-aws-lambda-function-with-terraform-custom-dependencies-7874407cd4fc
+###############################################################################
+resource "null_resource" "package_nlp_layer" {
+  triggers = {
+    redeployment = sha1(jsonencode([
+      "${path.module}/lambda_layer.tf",
+      file("${local.nlp_layer_packaging_script}"),
+      file("${local.nlp_layer_source_directory}/Dockerfile"),
+      file("${local.nlp_layer_source_directory}/create_container.sh"),
+      file("${local.nlp_layer_source_directory}/requirements.txt"),
+      fileexists("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") ? filebase64("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") : null
+    ]))
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash"]
+    command     = local.nlp_layer_packaging_script
+
+    environment = {
+      SOURCE_CODE_PATH = local.nlp_layer_source_directory
+      RUNTIME          = var.lambda_python_runtime
+      CONTAINER_NAME   = local.nlp_layer_name
+      PACKAGE_NAME     = local.nlp_layer_dist_package_name
+    }
+  }
+}
+
+resource "aws_lambda_layer_version" "nlp" {
+  filename                 = "${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}"
+  source_code_hash         = fileexists("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") ? filebase64sha256("${local.nlp_layer_source_directory}/${local.nlp_layer_dist_package_name}") : null
+  layer_name               = local.nlp_layer_slug
+  compatible_architectures = var.compatible_architectures
+  compatible_runtimes      = [var.lambda_python_runtime]
+  lifecycle {
+    create_before_destroy = true
+  }
+  depends_on = [
+    null_resource.package_nlp_layer
+  ]
+}

api/terraform/lambda_openai_function.tf

@@ -79,7 +79,7 @@ resource "aws_lambda_function" "lambda_openai_function" {
   architectures    = var.compatible_architectures
   filename         = data.archive_file.lambda_openai_function.output_path
   source_code_hash = data.archive_file.lambda_openai_function.output_base64sha256
-  layers           = [aws_lambda_layer_version.genai.arn]
+  layers           = [aws_lambda_layer_version.genai.arn, aws_lambda_layer_version.nlp.arn]
   tags             = var.tags
 
   environment {

api/terraform/python/layer_genai/Dockerfile

@@ -9,8 +9,5 @@ WORKDIR /var/task
 
 COPY requirements.txt .
 
-ENV PYTHONPATH "${PYTHONPATH}:python/lib/python3.11/site-packages"
-
 RUN yum install -y zip
 RUN pip install -r requirements.txt --target python/lib/python3.11/site-packages
-RUN python3 -m spacy download en_core_web_sm

api/terraform/python/layer_genai/requirements.txt

@@ -21,9 +21,3 @@ pydantic==2.5.3
 pydantic-settings==2.1.0
 python-dotenv==1.0.0
 python-hcl2==4.3.2
-
-# NLP requirements
-# --------------------------
-python-Levenshtein==0.23.0
-spacy==3.7.2
-pyyaml

api/terraform/python/layer_nlp/.gitignore

@@ -0,0 +1,3 @@
+*.zip
+venv
+archive

api/terraform/python/layer_nlp/Dockerfile

@@ -0,0 +1,16 @@
+# Use an AWS Lambda Python runtime as the base image
+# https://hub.docker.com/r/amazon/aws-lambda-python
+# ------------------------------------------------------
+
+# Stage 1: Build dependencies for x86_64
+FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.11
+
+WORKDIR /var/task
+
+COPY requirements.txt .
+
+ENV PYTHONPATH "${PYTHONPATH}:python/lib/python3.11/site-packages"
+
+RUN yum install -y zip
+RUN pip install -r requirements.txt --target python/lib/python3.11/site-packages
+RUN python3 -m spacy download en_core_web_sm

api/terraform/python/layer_nlp/README.md

@@ -0,0 +1,3 @@
+# AWS Lambda Layer for OpenAI/Langchain Lambdas
+
+This layer contains the combined pip requirements for Natural Language Processing features.

api/terraform/python/layer_nlp/create_container.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#------------------------------------------------------------------------------
+# written by: Lawrence McDaniel
+#             https://lawrencemcdaniel.com/
+#
+# date:   nov-2023
+#
+# usage:  Lambda Python packaging tool.
+#         Called by Terraform "null_resource". Copies python
+#         module(s) plus any requirements to a dedicated folder so that
+#         it can be archived to a zip file for upload to
+#         AWS Lambda by Terraform.
+#------------------------------------------------------------------------------
+cd $SOURCE_CODE_PATH
+
+
+docker build -t $CONTAINER_NAME .
+docker rm $CONTAINER_NAME
+docker run --name $CONTAINER_NAME --entrypoint /bin/bash $CONTAINER_NAME -c "zip -r $PACKAGE_NAME ."
+
+# Delete the distribution package if it exists
+if [ -f $PACKAGE_NAME ]; then
+    rm $PACKAGE_NAME
+fi
+
+docker cp $CONTAINER_NAME:/var/task/$PACKAGE_NAME .

api/terraform/python/layer_nlp/requirements.txt

@@ -0,0 +1,15 @@
+# -----------------------------------------------------------------------------
+# written by: Lawrence McDaniel
+#             https://lawrencemcdaniel.com
+#
+# usage:      Shared Python requirements for AWS Lambda functions.
+#             Create a virtual environment in the root of this repository
+#             named `venv`. Terraform modules will look for and include these
+#             requirements in the zip package for this layer.
+# -----------------------------------------------------------------------------
+
+# NLP requirements
+# --------------------------
+python-Levenshtein==0.23.0
+spacy==3.7.2
+pyyaml