Add spark work group

[Avinash-1394]

Description

Add a spark work group in the same catalog as the current work group that executes functional tests.

Additional information

Preferred name of the work group: spark
Preferred engine config: lowest possible
Preferred session timeout: 1 hour.

Note: Do not create them in multiple catalogs since Athena spark does not have multi catalog support yet.

[dacort]

I recently implemented this in CDK - feel free to use my IAM role as a reference:

export interface AthenaStackProps extends cdk.StackProps {
  readonly bucket: s3.IBucket;
}

export class AthenaStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props: AthenaStackProps) {
    super(scope, id, props);

    // Athena infra and resources
    const athenaSparkName = "spark-poc";

    // Athena IAM role for both SQL and Spark workgroups
    const athenaRole = new iam.Role(this, "aws-poc-athena-role", {
      assumedBy: new iam.CompositePrincipal(
        new iam.ServicePrincipal("athena.amazonaws.com"),
        new iam.AccountPrincipal(this.account).withConditions({
          StringEquals: { "aws:SourceAccount": this.account },
          ArnLike: { "aws:SourceArn": `arn:aws:athena:${this.region}:${this.account}:workgroup/${athenaSparkName}` },
        })
      ),
      managedPolicies: [
        // iam.ManagedPolicy.fromAwsManagedPolicyName("AmazonS3ReadOnlyAccess"),
      ],
      inlinePolicies: {
        athenaDefaultSparkPolicy: new iam.PolicyDocument({
          statements: [
            new iam.PolicyStatement({
              actions: ["s3:PutObject", "s3:ListBucket", "s3:DeleteObject", "s3:GetObject"],
              resources: [props.bucket.bucketArn, props.bucket.arnForObjects("*")],
            }),
            new iam.PolicyStatement({
              actions: [
                "athena:GetWorkGroup",
                "athena:TerminateSession",
                "athena:GetSession",
                "athena:GetSessionStatus",
                "athena:ListSessions",
                "athena:StartCalculationExecution",
                "athena:GetCalculationExecutionCode",
                "athena:StopCalculationExecution",
                "athena:ListCalculationExecutions",
                "athena:GetCalculationExecution",
                "athena:GetCalculationExecutionStatus",
                "athena:ListExecutors",
                "athena:ExportNotebook",
                "athena:UpdateNotebook",
              ],
              resources: [`arn:aws:athena:${this.region}:${this.account}:workgroup/${athenaSparkName}`],
            }),
            new iam.PolicyStatement({
              actions: ["logs:CreateLogStream", "logs:DescribeLogStreams", "logs:CreateLogGroup", "logs:PutLogEvents"],
              resources: [
                `arn:aws:logs:${this.region}:${this.account}:log-group:/aws-athena:*`,
                `arn:aws:logs:${this.region}:${this.account}:log-group:/aws-athena*:log-stream:*`,
              ],
            }),
            new iam.PolicyStatement({
              actions: ["logs:DescribeLogGroups"],
              resources: [`arn:aws:logs:${this.region}:${this.account}:log-group::*`],
            }),
            new iam.PolicyStatement({
              actions: ["cloudwatch:PutMetricData"],
              resources: ["*"],
              conditions: { StringEquals: { "cloudwatch:namespace": "AmazonAthenaForApacheSpark" } },
            }),
          ],
        }),
      },
    });

    // Athena Spark Workgroup
    const athenaSpark = new athena.CfnWorkGroup(this, "aws-poc-athena-spark-workgroup", {
      name: athenaSparkName,
      recursiveDeleteOption: true,
      workGroupConfiguration: {
        engineVersion: {
          selectedEngineVersion: "PySpark engine version 3",
        },
        resultConfiguration: {
          outputLocation: `s3://${props.bucket.bucketName}/athena/results/spark/`,
        },
        executionRole: athenaRole.roleArn,
      },
    });
  }
}