From 2104d98f34c26f05d80d2076b841197ad56b8e6b Mon Sep 17 00:00:00 2001 From: Julio Faerman <356476+faermanj@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:12:35 +0000 Subject: [PATCH] wip --- Dockerfile | 4 + aws/cloudtrail/data-copy.sh | 1 + aws/cloudtrail/data-reset.sh | 1 + aws/cloudtrail/index.ipynb | 577 ++++++++++++++++++ aws/{ => ocp}/create-cluster.aws-default.sh | 0 .../create-cluster.aws-singlenode.sh | 0 .../create-cluster.aws-singletiny.sh | 0 aws/{ => ocp}/create-cluster.aws-spot.sh | 0 .../install-config.aws-default.env.yaml | 0 .../install-config.aws-singlenode.env.yaml | 0 .../install-config.aws-singletiny.env.yaml | 0 .../install-config.aws-spot.env.yaml | 0 12 files changed, 583 insertions(+) create mode 100644 aws/cloudtrail/data-copy.sh create mode 100644 aws/cloudtrail/data-reset.sh create mode 100644 aws/cloudtrail/index.ipynb rename aws/{ => ocp}/create-cluster.aws-default.sh (100%) rename aws/{ => ocp}/create-cluster.aws-singlenode.sh (100%) rename aws/{ => ocp}/create-cluster.aws-singletiny.sh (100%) rename aws/{ => ocp}/create-cluster.aws-spot.sh (100%) rename aws/{ => ocp}/install-config.aws-default.env.yaml (100%) rename aws/{ => ocp}/install-config.aws-singlenode.env.yaml (100%) rename aws/{ => ocp}/install-config.aws-singletiny.env.yaml (100%) rename aws/{ => ocp}/install-config.aws-spot.env.yaml (100%) diff --git a/Dockerfile b/Dockerfile index 4389857..311052d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,10 @@ RUN bash -c "chmod a+x $SDKMAN_DIR/bin/sdkman-init.sh \ && sdk install quarkus \ && sdk install maven \ " +# Python +#update pip +RUN bash -c "pip install --upgrade pip" +RUN bash -c "pip install pandas jupyter" # Done :) # RUN bash -c "echo 'done.'" diff --git a/aws/cloudtrail/data-copy.sh b/aws/cloudtrail/data-copy.sh new file mode 100644 index 0000000..0427b75 --- /dev/null +++ b/aws/cloudtrail/data-copy.sh @@ -0,0 +1 @@ +aws s3 sync s3://jufaerma-ap-northeast-1 ./.data \ No newline at end of file diff --git a/aws/cloudtrail/data-reset.sh b/aws/cloudtrail/data-reset.sh new file mode 100644 index 0000000..0427b75 --- /dev/null +++ b/aws/cloudtrail/data-reset.sh @@ -0,0 +1 @@ +aws s3 sync s3://jufaerma-ap-northeast-1 ./.data \ No newline at end of file diff --git a/aws/cloudtrail/index.ipynb b/aws/cloudtrail/index.ipynb new file mode 100644 index 0000000..b961c21 --- /dev/null +++ b/aws/cloudtrail/index.ipynb @@ -0,0 +1,577 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting pandas\n", + " Downloading pandas-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n", + "Collecting numpy<2,>=1.26.0 (from pandas)\n", + " Downloading numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.0/61.0 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.2 in /home/gitpod/.pyenv/versions/3.12.1/lib/python3.12/site-packages (from pandas) (2.8.2)\n", + "Collecting pytz>=2020.1 (from pandas)\n", + " Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n", + "Collecting tzdata>=2022.7 (from pandas)\n", + " Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n", + "Requirement already satisfied: six>=1.5 in /home/gitpod/.pyenv/versions/3.12.1/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n", + "Downloading pandas-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.7 MB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.7/12.7 MB\u001b[0m \u001b[31m113.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m01\u001b[0m:01\u001b[0m\n", + "\u001b[?25hDownloading numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.0 MB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.0/18.0 MB\u001b[0m \u001b[31m118.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m505.5/505.5 kB\u001b[0m \u001b[31m114.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m345.4/345.4 kB\u001b[0m \u001b[31m94.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: pytz, tzdata, numpy, pandas\n", + "Successfully installed numpy-1.26.4 pandas-2.2.1 pytz-2024.1 tzdata-2024.1\n" + ] + } + ], + "source": [ + "! pip install pandas" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['./.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2025Z_X1c9NiKwnV3CYmXI.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1950Z_Qk2aq2jark4HpBlG.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2040Z_Vmmj0B18ci4zTuy2.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1945Z_SsNqCRTyws7xrY3J.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1955Z_0JYT7c5H38fS0Jzy.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1950Z_yzePA9EprIlgYK2R.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2000Z_v34LxJHzVlXIw8cr.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1955Z_et04WAteFcLN9YUu.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2000Z_yY6F9bSjH8QBAehk.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2000Z_PGu8Zq1rN9hTTm4k.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2000Z_zDdght9V3fwyY23H.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2010Z_GiGEsRw2KfolpslM.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2010Z_RJwgNb5JnrV3W9xv.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2015Z_SeCEZLsne6devXoH.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2020Z_t4m3yswisWjpr4o9.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2010Z_UaQ0DZP4OnkjJDzc.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2015Z_pDJdSu7mMtS0AIdw.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2025Z_PjbFu4ln25Zq2f3Z.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2030Z_YkevR3m6nigurLLv.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2035Z_HfhEvDRDJKxOx5MC.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2035Z_hJZ2XQfeaAiV60XT.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2015Z_yOFUUoAdRpjUVbV0.json.gz']" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "import pandas as pd\n", + "import json\n", + "\n", + "datafiles = []\n", + "for root, dirs, files in os.walk(\"./.data\"):\n", + " for file in files:\n", + " if file.endswith(\".json.gz\"):\n", + " datafiles.append(os.path.join(root, file))\n", + "\n", + "datafiles" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eventVersioneventTimeeventSourceeventNameawsRegionsourceIPAddressuserAgentrequestIDeventIDreadOnly...requestParameters.DescribeVpcEndpointsRequestrequestParameters.securityGroupIdSet.itemsrequestParameters.vpcSet.itemsrequestParameters.groupIdrequestParameters.dhcpOptionsIdrequestParameters.routeTableIdSet.itemsrequestParameters.associationIdrequestParameters.networkAclIdSet.itemsrequestParameters.subnetIdrequestParameters.vpcId
01.082024-03-12T20:17:07Zsts.amazonaws.comAssumeRoleap-northeast-1access-analyzer.amazonaws.comaccess-analyzer.amazonaws.com68977b77-a50e-48db-a53e-2beb90ff8661b6ea10db-360e-3886-9d24-5764f2a8a98fTrue...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
11.092024-03-12T20:17:07Zs3.amazonaws.comGetBucketAclap-northeast-1cloudtrail.amazonaws.comcloudtrail.amazonaws.comS29Z263Y106DP0YMf527b186-aa7a-46e0-b555-444960378d77True...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
21.082024-03-12T20:17:09Zsts.amazonaws.comAssumeRoleap-northeast-1access-analyzer.amazonaws.comaccess-analyzer.amazonaws.com4c7307cf-657a-4f65-8fcf-155f9b306b80b8483928-1548-3b93-a9ef-a8f46aad3387True...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
31.082024-03-12T20:17:11Zsts.amazonaws.comAssumeRoleap-northeast-1access-analyzer.amazonaws.comaccess-analyzer.amazonaws.com790c1a5a-d727-4afa-8f3a-6c85689056441676c595-2c4b-386d-ae73-4f96eb555bb1True...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
41.092024-03-12T20:17:11Zs3.amazonaws.comGetBucketAclap-northeast-1cloudtrail.amazonaws.comcloudtrail.amazonaws.comDN585QA06J42CTZM7dfcc55f-6410-4ecc-b222-a026b7b65101True...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", + "

5 rows × 113 columns

\n", + "
" + ], + "text/plain": [ + " eventVersion eventTime eventSource eventName \\\n", + "0 1.08 2024-03-12T20:17:07Z sts.amazonaws.com AssumeRole \n", + "1 1.09 2024-03-12T20:17:07Z s3.amazonaws.com GetBucketAcl \n", + "2 1.08 2024-03-12T20:17:09Z sts.amazonaws.com AssumeRole \n", + "3 1.08 2024-03-12T20:17:11Z sts.amazonaws.com AssumeRole \n", + "4 1.09 2024-03-12T20:17:11Z s3.amazonaws.com GetBucketAcl \n", + "\n", + " awsRegion sourceIPAddress \\\n", + "0 ap-northeast-1 access-analyzer.amazonaws.com \n", + "1 ap-northeast-1 cloudtrail.amazonaws.com \n", + "2 ap-northeast-1 access-analyzer.amazonaws.com \n", + "3 ap-northeast-1 access-analyzer.amazonaws.com \n", + "4 ap-northeast-1 cloudtrail.amazonaws.com \n", + "\n", + " userAgent requestID \\\n", + "0 access-analyzer.amazonaws.com 68977b77-a50e-48db-a53e-2beb90ff8661 \n", + "1 cloudtrail.amazonaws.com S29Z263Y106DP0YM \n", + "2 access-analyzer.amazonaws.com 4c7307cf-657a-4f65-8fcf-155f9b306b80 \n", + "3 access-analyzer.amazonaws.com 790c1a5a-d727-4afa-8f3a-6c8568905644 \n", + "4 cloudtrail.amazonaws.com DN585QA06J42CTZM \n", + "\n", + " eventID readOnly ... \\\n", + "0 b6ea10db-360e-3886-9d24-5764f2a8a98f True ... \n", + "1 f527b186-aa7a-46e0-b555-444960378d77 True ... \n", + "2 b8483928-1548-3b93-a9ef-a8f46aad3387 True ... \n", + "3 1676c595-2c4b-386d-ae73-4f96eb555bb1 True ... \n", + "4 7dfcc55f-6410-4ecc-b222-a026b7b65101 True ... \n", + "\n", + " requestParameters.DescribeVpcEndpointsRequest \\\n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + " requestParameters.securityGroupIdSet.items requestParameters.vpcSet.items \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " requestParameters.groupId requestParameters.dhcpOptionsId \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " requestParameters.routeTableIdSet.items requestParameters.associationId \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " requestParameters.networkAclIdSet.items requestParameters.subnetId \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " requestParameters.vpcId \n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "\n", + "[5 rows x 113 columns]" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Reading all CSV files into a list of dataframes\n", + "dataframes = [pd.read_json(file) for file in datafiles]\n", + "\n", + "# Combining all dataframes into a single dataframe\n", + "dff = pd.concat(dataframes, ignore_index=True)\n", + "df = [pd.json_normalize(record) for record in dff['Records']]\n", + "df = pd.concat(df, ignore_index=True)\n", + "\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
eventVersioneventTimeeventSourceeventNameawsRegionsourceIPAddressuserAgentrequestIDeventIDreadOnly...requestParameters.vpcSet.itemsrequestParameters.groupIdrequestParameters.dhcpOptionsIdrequestParameters.routeTableIdSet.itemsrequestParameters.associationIdrequestParameters.networkAclIdSet.itemsrequestParameters.subnetIdrequestParameters.vpcIdserviceapi_call
\n", + "

0 rows × 115 columns

\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [eventVersion, eventTime, eventSource, eventName, awsRegion, sourceIPAddress, userAgent, requestID, eventID, readOnly, resources, eventType, managementEvent, recipientAccountId, sharedEventID, eventCategory, userIdentity.type, userIdentity.invokedBy, requestParameters.roleArn, requestParameters.roleSessionName, responseElements.credentials.accessKeyId, responseElements.credentials.sessionToken, responseElements.credentials.expiration, responseElements.assumedRoleUser.assumedRoleId, responseElements.assumedRoleUser.arn, responseElements, requestParameters.bucketName, requestParameters.Host, requestParameters.acl, additionalEventData.SignatureVersion, additionalEventData.CipherSuite, additionalEventData.bytesTransferredIn, additionalEventData.AuthenticationMethod, additionalEventData.x-amz-id-2, additionalEventData.bytesTransferredOut, errorCode, errorMessage, apiVersion, userIdentity.principalId, userIdentity.arn, userIdentity.accountId, userIdentity.accessKeyId, userIdentity.sessionContext.sessionIssuer.type, userIdentity.sessionContext.sessionIssuer.principalId, userIdentity.sessionContext.sessionIssuer.arn, userIdentity.sessionContext.sessionIssuer.accountId, userIdentity.sessionContext.sessionIssuer.userName, userIdentity.sessionContext.attributes.creationDate, userIdentity.sessionContext.attributes.mfaAuthenticated, requestParameters.fileSystemId, tlsDetails.tlsVersion, tlsDetails.cipherSuite, tlsDetails.clientProvidedHostHeader, vpcEndpointId, requestParameters.durationSeconds, requestParameters.filterSet.items, requestParameters.DescribeLaunchTemplatesRequest.Filter.Value.tag, requestParameters.DescribeLaunchTemplatesRequest.Filter.Value.content, requestParameters.DescribeLaunchTemplatesRequest.Filter.tag, requestParameters.DescribeLaunchTemplatesRequest.Filter.Name, requestParameters.ownersSet.items, requestParameters.retiringPrincipal, requestParameters.GetEbsDefaultKmsKeyIdRequest, requestParameters.trailNameList, requestParameters.resourceIdList, requestParameters.policy, requestParameters.logging, requestParameters.lifecycle, requestParameters.notification, requestParameters.tagging, requestParameters.versioning, requestParameters.website, requestParameters.cors, requestParameters.requestPayment, requestParameters.accelerate, requestParameters.replication, requestParameters.encryption, requestParameters.publicAccessBlock, requestParameters.object-lock, userIdentity.userName, sessionCredentialFromConsole, requestParameters.includeShadowTrails, requestParameters.name, requestParameters, requestParameters.trailName, requestParameters.Type, requestParameters.filter.status.eq, requestParameters.sort.attributeName, requestParameters.sort.orderBy, requestParameters.analyzerArn, requestParameters.analyzerName, requestParameters.tableName, requestParameters.type, requestParameters.clientToken, responseElements.Access-Control-Expose-Headers, responseElements.message, requestParameters.nextToken, requestParameters.lookupAttributes, requestParameters.maxResults, requestParameters.eventCategory, ...]\n", + "Index: []\n", + "\n", + "[0 rows x 115 columns]" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[df['eventName'] == 'RunInstances'] " + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "api_call\n", + "access-analyzer:CreateAnalyzer 1\n", + "access-analyzer:GetAnalyzer 1\n", + "access-analyzer:GetFindingsStatistics 1\n", + "access-analyzer:ListAnalyzers 1\n", + "access-analyzer:ListArchiveRules 1\n", + "access-analyzer:ListFindings 4\n", + "autoscaling:DescribeAutoScalingGroups 1\n", + "batch:ListSchedulingPolicies 1\n", + "cloudformation:ListStacks 1\n", + "cloudtrail:DescribeTrails 8\n", + "cloudtrail:GetEventSelectors 9\n", + "cloudtrail:GetInsightSelectors 1\n", + "cloudtrail:GetTrailStatus 11\n", + "cloudtrail:ListEventDataStores 1\n", + "cloudtrail:ListTags 2\n", + "cloudtrail:LookupEvents 2\n", + "dynamodb:DescribeTable 9\n", + "ec2:DeleteDhcpOptions 1\n", + "ec2:DeleteSecurityGroup 1\n", + "ec2:DeleteSubnet 3\n", + "ec2:DeleteVpc 1\n", + "ec2:DescribeAddresses 1\n", + "ec2:DescribeDhcpOptions 1\n", + "ec2:DescribeImages 1\n", + "ec2:DescribeInstances 16\n", + "ec2:DescribeInternetGateways 1\n", + "ec2:DescribeLaunchTemplates 5\n", + "ec2:DescribeNatGateways 1\n", + "ec2:DescribeNetworkAcls 2\n", + "ec2:DescribeNetworkInterfaces 1\n", + "ec2:DescribePlacementGroups 1\n", + "ec2:DescribeRouteTables 2\n", + "ec2:DescribeSecurityGroups 3\n", + "ec2:DescribeSnapshots 6\n", + "ec2:DescribeSubnets 1\n", + "ec2:DescribeVolumes 6\n", + "ec2:DescribeVpcEndpoints 1\n", + "ec2:DescribeVpcPeeringConnections 1\n", + "ec2:DescribeVpcs 2\n", + "ec2:DisassociateRouteTable 1\n", + "ec2:GetEbsDefaultKmsKeyId 5\n", + "elasticbeanstalk:DescribeApplicationVersions 1\n", + "elasticfilesystem:DeleteFileSystem 1\n", + "elasticfilesystem:DescribeFileSystems 1\n", + "elasticloadbalancing:DescribeLoadBalancers 2\n", + "elasticloadbalancing:DescribeTargetGroups 1\n", + "kms:ListRetirableGrants 5\n", + "lambda:ListFunctions20150331 1\n", + "notifications:ListNotificationHubs 4\n", + "rds:DescribeDBClusterParameterGroups 1\n", + "rds:DescribeDBClusterSnapshots 1\n", + "rds:DescribeDBClusters 1\n", + "rds:DescribeDBInstances 1\n", + "rds:DescribeDBProxies 1\n", + "rds:DescribeDBSnapshots 1\n", + "rds:DescribeDBSubnetGroups 1\n", + "resource-explorer-2:ListIndexes 1\n", + "s3:GetAccelerateConfiguration 1\n", + "s3:GetAccountPublicAccessBlock 1\n", + "s3:GetBucketAcl 140\n", + "s3:GetBucketCors 1\n", + "s3:GetBucketEncryption 1\n", + "s3:GetBucketLifecycle 1\n", + "s3:GetBucketLogging 1\n", + "s3:GetBucketNotification 1\n", + "s3:GetBucketObjectLockConfiguration 1\n", + "s3:GetBucketPolicy 2\n", + "s3:GetBucketPublicAccessBlock 1\n", + "s3:GetBucketReplication 1\n", + "s3:GetBucketRequestPayment 1\n", + "s3:GetBucketTagging 1\n", + "s3:GetBucketVersioning 1\n", + "s3:GetBucketWebsite 1\n", + "s3:ListAccessPoints 1\n", + "s3:ListBuckets 3\n", + "sts:AssumeRole 91\n", + "sts:GetCallerIdentity 3\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.set_option('display.max_rows', None)\n", + "df['service'] = df['eventSource'].str.replace('.amazonaws.com', '', regex=False)\n", + "df['api_call'] = df['service'] + ':' + df['eventName']\n", + "\n", + "\n", + "api_call_counts = df['api_call'].value_counts().sort_index()\n", + "\n", + "api_call_counts" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [], + "source": [ + "output_file_path_counts = 'api_calls_counts.txt'\n", + "\n", + "# Write the counts of each API call to a file\n", + "with open(output_file_path_counts, 'w') as file:\n", + " for api_call, count in api_call_counts.items():\n", + " file.write(f\"{api_call}: {count}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/aws/create-cluster.aws-default.sh b/aws/ocp/create-cluster.aws-default.sh similarity index 100% rename from aws/create-cluster.aws-default.sh rename to aws/ocp/create-cluster.aws-default.sh diff --git a/aws/create-cluster.aws-singlenode.sh b/aws/ocp/create-cluster.aws-singlenode.sh similarity index 100% rename from aws/create-cluster.aws-singlenode.sh rename to aws/ocp/create-cluster.aws-singlenode.sh diff --git a/aws/create-cluster.aws-singletiny.sh b/aws/ocp/create-cluster.aws-singletiny.sh similarity index 100% rename from aws/create-cluster.aws-singletiny.sh rename to aws/ocp/create-cluster.aws-singletiny.sh diff --git a/aws/create-cluster.aws-spot.sh b/aws/ocp/create-cluster.aws-spot.sh similarity index 100% rename from aws/create-cluster.aws-spot.sh rename to aws/ocp/create-cluster.aws-spot.sh diff --git a/aws/install-config.aws-default.env.yaml b/aws/ocp/install-config.aws-default.env.yaml similarity index 100% rename from aws/install-config.aws-default.env.yaml rename to aws/ocp/install-config.aws-default.env.yaml diff --git a/aws/install-config.aws-singlenode.env.yaml b/aws/ocp/install-config.aws-singlenode.env.yaml similarity index 100% rename from aws/install-config.aws-singlenode.env.yaml rename to aws/ocp/install-config.aws-singlenode.env.yaml diff --git a/aws/install-config.aws-singletiny.env.yaml b/aws/ocp/install-config.aws-singletiny.env.yaml similarity index 100% rename from aws/install-config.aws-singletiny.env.yaml rename to aws/ocp/install-config.aws-singletiny.env.yaml diff --git a/aws/install-config.aws-spot.env.yaml b/aws/ocp/install-config.aws-spot.env.yaml similarity index 100% rename from aws/install-config.aws-spot.env.yaml rename to aws/ocp/install-config.aws-spot.env.yaml