p3.12 builds (#377)

Builds for Python3.12, removed Python 3.8 and Python 3.9
keithrozario · Jan 3, 2024 · dea1a6a · dea1a6a
1 parent 8363869
commit dea1a6a
Show file tree

Hide file tree

Showing 27 changed files with 152 additions and 8,913 deletions.
diff --git a/.github/workflows/container_builds.yml b/.github/workflows/container_builds.yml
@@ -17,9 +17,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - python_version: p39
-            arch: x86
-            build_platform: linux/amd64
           - python_version: p310
             arch: x86
             build_platform: linux/amd64
@@ -32,6 +29,12 @@ jobs:
           - python_version: p311
             arch: arm64
             build_platform: linux/arm64
+          - python_version: p312
+            arch: x86
+            build_platform: linux/amd64
+          - python_version: p312
+            arch: arm64
+            build_platform: linux/arm64
 
 
     permissions:

diff --git a/pipeline/Serverless/02_pipeline/pipeline.yml b/pipeline/Serverless/02_pipeline/pipeline.yml
@@ -9,35 +9,6 @@ check:
     - arn:aws:lambda:${self:provider.region}:770693421928:layer:Klayers-p38-requests:7
     - arn:aws:lambda:${self:provider.region}:017000801446:layer:AWSLambdaPowertoolsPython:6
 
-build:
-  handler: 02_pipeline/build.main
-  description: Builds the package inside the lambda using pip
-  runtime: python3.8
-  timeout: 180
-  memorySize: 1769
-  environment:
-    POWERTOOLS_SERVICE_NAME: Klayers.Build
-  iamRoleStatementsName: ${self:provider.stage}-build
-  iamRoleStatements:
-  - Effect: Allow
-    Action:
-    - s3:PutObject
-    - s3:PutObjectAcl
-    Resource: ${self:custom.s3LayersArn}/*
-  - Effect: Allow
-    Action:
-    - s3:ListBucket
-    Resource: ${self:custom.s3LayersArn}
-  - Effect: Allow
-    Action:
-    - dynamodb:PutItem
-    - dynamodb:UpdateItem
-    - dynamodb:GetItem
-    Resource: ${self:custom.dbArn}
-  layers:
-    - arn:aws:lambda:${self:provider.region}:553035198032:layer:gcc-lambda2:4  # https://github.com/lambci/gcc-lambda-layer
-    - arn:aws:lambda:${self:provider.region}:017000801446:layer:AWSLambdaPowertoolsPython:6
-
 deploy:
   handler: 02_pipeline/deploy.main
   description: Deploys functions to multiple regions

diff --git a/pipeline/Serverless/02_pipeline/pipeline_build_containers.yml b/pipeline/Serverless/02_pipeline/pipeline_build_containers.yml
@@ -1,9 +1,9 @@
-build39:
-  description: Builds the package in a container for python3.9 then uploads to S3
-  image: ${ssm:/${self:service}/${self:provider.stage}/build/p39/x86/repo}@${ssm:/${self:service}/${self:provider.stage}/build/p39/x86/digest}
+build310:
+  description: Builds the package in a container for python3.10 then uploads to S3
+  image: ${ssm:/${self:service}/${self:provider.stage}/build/p310/x86/repo}@${ssm:/${self:service}/${self:provider.stage}/build/p310/x86/digest}
   timeout: 600
   memorySize: 1769  # exactly one vcpu
-  iamRoleStatementsName: ${self:provider.stage}-buildp39
+  iamRoleStatementsName: ${self:provider.stage}-buildp310
   iamRoleStatements:
   - Effect: Allow
     Action:
@@ -21,12 +21,12 @@ build39:
     - dynamodb:GetItem
     Resource: ${self:custom.dbArn}
 
-build310:
-  description: Builds the package in a container for python3.10 then uploads to S3
-  image: ${ssm:/${self:service}/${self:provider.stage}/build/p310/x86/repo}@${ssm:/${self:service}/${self:provider.stage}/build/p310/x86/digest}
+build311:
+  description: Builds the package in a container for python3.11 then uploads to S3
+  image: ${ssm:/${self:service}/${self:provider.stage}/build/p311/x86/repo}@${ssm:/${self:service}/${self:provider.stage}/build/p311/x86/digest}
   timeout: 600
   memorySize: 1769  # exactly one vcpu
-  iamRoleStatementsName: ${self:provider.stage}-buildp310
+  iamRoleStatementsName: ${self:provider.stage}-buildp311
   iamRoleStatements:
   - Effect: Allow
     Action:
@@ -44,12 +44,12 @@ build310:
     - dynamodb:GetItem
     Resource: ${self:custom.dbArn}
 
-build311:
-  description: Builds the package in a container for python3.11 then uploads to S3
-  image: ${ssm:/${self:service}/${self:provider.stage}/build/p311/x86/repo}@${ssm:/${self:service}/${self:provider.stage}/build/p311/x86/digest}
+build312:
+  description: Builds the package in a container for python3.12 then uploads to S3
+  image: ${ssm:/${self:service}/${self:provider.stage}/build/p312/x86/repo}@${ssm:/${self:service}/${self:provider.stage}/build/p312/x86/digest}
   timeout: 600
   memorySize: 1769  # exactly one vcpu
-  iamRoleStatementsName: ${self:provider.stage}-buildp311
+  iamRoleStatementsName: ${self:provider.stage}-buildp312
   iamRoleStatements:
   - Effect: Allow
     Action:
@@ -99,6 +99,30 @@ build311Arm64:
   memorySize: 1769  # exactly one vcpu
   iamRoleStatementsName: ${self:provider.stage}-buildp311arm64
   iamRoleStatements:
+  - Effect: Allow
+    Action:
+    - s3:PutObject
+    - s3:PutObjectAcl
+    Resource: ${self:custom.s3LayersArn}/*
+  - Effect: Allow
+    Action:
+    - s3:ListBucket
+    Resource: ${self:custom.s3LayersArn}
+  - Effect: Allow
+    Action:
+    - dynamodb:PutItem
+    - dynamodb:UpdateItem
+    - dynamodb:GetItem
+    Resource: ${self:custom.dbArn}
+  architecture: arm64
+
+build312Arm64:
+  description: Builds the package in a container for python3.12 then uploads to S3
+  image: ${ssm:/${self:service}/${self:provider.stage}/build/p312/arm64/repo}@${ssm:/${self:service}/${self:provider.stage}/build/p312/arm64/digest}
+  timeout: 600
+  memorySize: 1769  # exactly one vcpu
+  iamRoleStatementsName: ${self:provider.stage}-buildp312arm64
+  iamRoleStatements:
   - Effect: Allow
     Action:
     - s3:PutObject

diff --git a/pipeline/Serverless/state_machines/02_pipeline.yml b/pipeline/Serverless/state_machines/02_pipeline.yml
@@ -29,32 +29,33 @@ definition:
     ChoicePythonVersion:
       Type: Choice
       Choices:
-        - Variable: $.python_version
-          StringEquals: "p3.8"
-          Next: Build38
-        - Variable: $.python_version
-          StringEquals: "p3.9"
-          Next: Build39
         - Variable: $.python_version
           StringEquals: "p3.10"
           Next: Build310
         - Variable: $.python_version
           StringEquals: "p3.11"
           Next: Build311
+        - Variable: $.python_version
+          StringEquals: "p3.12"
+          Next: Build312
         - Variable: $.python_version
           StringEquals: "p3.10-arm64"
           Next: Build310arm64
         - Variable: $.python_version
           StringEquals: "p3.11-arm64"
           Next: Build311arm64
+        - Variable: $.python_version
+          StringEquals: "p3.12-arm64"
+          Next: Build312arm64
       Default: UnknownPythonVersion
-    Build38:
+    Build310:
       Type: Task
       Resource:
-        Fn::GetAtt: [BuildLambdaFunction, Arn]
+        Fn::GetAtt: [Build310LambdaFunction, Arn]
       Next: Deploy
       Retry:
         - ErrorEquals:
+          - Lambda.AWSLambdaException
           - States.Timeout
           - Lambda.ClientExecutionTimeoutException
           - Lambda.ServiceException
@@ -67,10 +68,10 @@ definition:
         - ErrorEquals:
           - States.ALL
           Next: BuildFail
-    Build39:
+    Build311:
       Type: Task
       Resource:
-        Fn::GetAtt: [Build39LambdaFunction, Arn]
+        Fn::GetAtt: [Build311LambdaFunction, Arn]
       Next: Deploy
       Retry:
         - ErrorEquals:
@@ -87,10 +88,10 @@ definition:
         - ErrorEquals:
           - States.ALL
           Next: BuildFail
-    Build310:
+    Build312:
       Type: Task
       Resource:
-        Fn::GetAtt: [Build310LambdaFunction, Arn]
+        Fn::GetAtt: [Build312LambdaFunction, Arn]
       Next: Deploy
       Retry:
         - ErrorEquals:
@@ -107,14 +108,13 @@ definition:
         - ErrorEquals:
           - States.ALL
           Next: BuildFail
-    Build311:
+    Build310arm64:
       Type: Task
       Resource:
-        Fn::GetAtt: [Build311LambdaFunction, Arn]
+        Fn::GetAtt: [Build310Arm64LambdaFunction, Arn]
       Next: Deploy
       Retry:
         - ErrorEquals:
-          - Lambda.AWSLambdaException
           - States.Timeout
           - Lambda.ClientExecutionTimeoutException
           - Lambda.ServiceException
@@ -127,10 +127,10 @@ definition:
         - ErrorEquals:
           - States.ALL
           Next: BuildFail
-    Build310arm64:
+    Build311arm64:
       Type: Task
       Resource:
-        Fn::GetAtt: [Build310Arm64LambdaFunction, Arn]
+        Fn::GetAtt: [Build311Arm64LambdaFunction, Arn]
       Next: Deploy
       Retry:
         - ErrorEquals:
@@ -146,10 +146,10 @@ definition:
         - ErrorEquals:
           - States.ALL
           Next: BuildFail
-    Build311arm64:
+    Build312arm64:
       Type: Task
       Resource:
-        Fn::GetAtt: [Build311Arm64LambdaFunction, Arn]
+        Fn::GetAtt: [Build312Arm64LambdaFunction, Arn]
       Next: Deploy
       Retry:
         - ErrorEquals:

diff --git a/pipeline/Terraform/container_repositories.tf b/pipeline/Terraform/container_repositories.tf
@@ -1,24 +1,5 @@
 # Container Build Images
 
-## Python 3.9 - x86 (only)
-resource "aws_ecr_repository" "p39build_x86" {
-  name                 = "p39build"
-  image_tag_mutability = "MUTABLE"
-  force_delete         = true
-
-  image_scanning_configuration {
-    scan_on_push = true
-  }
-}
-
-resource "aws_ssm_parameter" "p39_build_repo" {
-  type        = "String"
-  description = "URL for p39 x86 repo"
-  name        = "/${var.app_name}/${local.workspace_full_name}/build/p39/x86/repo"
-  value       = aws_ecr_repository.p39build_x86.repository_url
-  overwrite   = true
-}
-
 ## Python 3.10 - x86
 resource "aws_ecr_repository" "p310build_x86" {
   name                 = "p310build"
@@ -95,3 +76,21 @@ resource "aws_ssm_parameter" "p311_arm64_build_repo" {
   overwrite   = true
 }
 
+
+## Python 3.12 builds
+
+module "python312_x86_build" {
+  source             = "./container_repository"
+  app_name           = var.app_name
+  workspace_full_name = local.workspace_full_name
+  python_version = "p312"
+  architecture = "x86"
+}
+
+module "python312_arm64_build" {
+  source             = "./container_repository"
+  app_name           = var.app_name
+  workspace_full_name = local.workspace_full_name
+  python_version = "p312"
+  architecture = "arm64"
+}
diff --git a/pipeline/Terraform/container_repository/main.tf b/pipeline/Terraform/container_repository/main.tf
@@ -0,0 +1,23 @@
+# Container Build Images
+variable app_name{}
+variable workspace_full_name{}
+variable python_version{}
+variable architecture{}
+
+resource "aws_ecr_repository" "build" {
+  name                 = "${var.python_version}${var.architecture}build"
+  image_tag_mutability = "MUTABLE"
+  force_delete         = true
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+}
+
+resource "aws_ssm_parameter" "build" {
+  type        = "String"
+  description = "URL for ${var.python_version} ${var.architecture} repo"
+  name        = "/${var.app_name}/${var.workspace_full_name}/build/${var.python_version}/${var.architecture}/repo"
+  value       = aws_ecr_repository.build.repository_url
+  overwrite   = true
+}
diff --git a/pipeline/config/config.json b/pipeline/config/config.json
@@ -1,21 +1,21 @@
 {
-    "python_versions": ["p3.8", "p3.9", "p3.10", "p3.10-arm64", "p3.11", "p3.11-arm64"],
-    "p3.8": {
-        "packages_file": "packages_p38.csv"
-    },
-    "p3.9": {
-        "packages_file": "packages_p39.csv"
-    },
+    "python_versions": ["p3.10", "p3.10-arm64", "p3.11", "p3.11-arm64", "p3.12", "p3.12-arm64"],
     "p3.10": {
         "packages_file": "packages_p310.csv"
     },
     "p3.11":{
         "packages_file": "packages_p311.csv"
     },
+    "p3.12":{
+        "packages_file": "packages_p312.csv"
+    },
     "p3.10-arm64": {
         "packages_file": "packages_p310-arm64.csv"
     },
     "p3.11-arm64":{
         "packages_file": "packages_p311-arm64.csv"
+    },
+    "p3.12-arm64":{
+        "packages_file": "packages_p312-arm64.csv"
     }
 }
diff --git a/pipeline/config/packages_p312-arm64.csv b/pipeline/config/packages_p312-arm64.csv
@@ -0,0 +1 @@
+Package_Name,License,Authors/Maintainers
diff --git a/pipeline/config/packages_p312.csv b/pipeline/config/packages_p312.csv
@@ -0,0 +1 @@
+Package_Name,License,Authors/Maintainers