From 2104d98f34c26f05d80d2076b841197ad56b8e6b Mon Sep 17 00:00:00 2001 From: Julio Faerman <356476+faermanj@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:12:35 +0000 Subject: [PATCH] wip --- Dockerfile | 4 + aws/cloudtrail/data-copy.sh | 1 + aws/cloudtrail/data-reset.sh | 1 + aws/cloudtrail/index.ipynb | 577 ++++++++++++++++++ aws/{ => ocp}/create-cluster.aws-default.sh | 0 .../create-cluster.aws-singlenode.sh | 0 .../create-cluster.aws-singletiny.sh | 0 aws/{ => ocp}/create-cluster.aws-spot.sh | 0 .../install-config.aws-default.env.yaml | 0 .../install-config.aws-singlenode.env.yaml | 0 .../install-config.aws-singletiny.env.yaml | 0 .../install-config.aws-spot.env.yaml | 0 12 files changed, 583 insertions(+) create mode 100644 aws/cloudtrail/data-copy.sh create mode 100644 aws/cloudtrail/data-reset.sh create mode 100644 aws/cloudtrail/index.ipynb rename aws/{ => ocp}/create-cluster.aws-default.sh (100%) rename aws/{ => ocp}/create-cluster.aws-singlenode.sh (100%) rename aws/{ => ocp}/create-cluster.aws-singletiny.sh (100%) rename aws/{ => ocp}/create-cluster.aws-spot.sh (100%) rename aws/{ => ocp}/install-config.aws-default.env.yaml (100%) rename aws/{ => ocp}/install-config.aws-singlenode.env.yaml (100%) rename aws/{ => ocp}/install-config.aws-singletiny.env.yaml (100%) rename aws/{ => ocp}/install-config.aws-spot.env.yaml (100%) diff --git a/Dockerfile b/Dockerfile index 4389857..311052d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,10 @@ RUN bash -c "chmod a+x $SDKMAN_DIR/bin/sdkman-init.sh \ && sdk install quarkus \ && sdk install maven \ " +# Python +#update pip +RUN bash -c "pip install --upgrade pip" +RUN bash -c "pip install pandas jupyter" # Done :) # RUN bash -c "echo 'done.'" diff --git a/aws/cloudtrail/data-copy.sh b/aws/cloudtrail/data-copy.sh new file mode 100644 index 0000000..0427b75 --- /dev/null +++ b/aws/cloudtrail/data-copy.sh @@ -0,0 +1 @@ +aws s3 sync s3://jufaerma-ap-northeast-1 ./.data \ No newline at end of file diff --git a/aws/cloudtrail/data-reset.sh b/aws/cloudtrail/data-reset.sh new file mode 100644 index 0000000..0427b75 --- /dev/null +++ b/aws/cloudtrail/data-reset.sh @@ -0,0 +1 @@ +aws s3 sync s3://jufaerma-ap-northeast-1 ./.data \ No newline at end of file diff --git a/aws/cloudtrail/index.ipynb b/aws/cloudtrail/index.ipynb new file mode 100644 index 0000000..b961c21 --- /dev/null +++ b/aws/cloudtrail/index.ipynb @@ -0,0 +1,577 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting pandas\n", + " Downloading pandas-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n", + "Collecting numpy<2,>=1.26.0 (from pandas)\n", + " Downloading numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.0/61.0 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.2 in /home/gitpod/.pyenv/versions/3.12.1/lib/python3.12/site-packages (from pandas) (2.8.2)\n", + "Collecting pytz>=2020.1 (from pandas)\n", + " Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n", + "Collecting tzdata>=2022.7 (from pandas)\n", + " Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n", + "Requirement already satisfied: six>=1.5 in /home/gitpod/.pyenv/versions/3.12.1/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n", + "Downloading pandas-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.7 MB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.7/12.7 MB\u001b[0m \u001b[31m113.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m01\u001b[0m:01\u001b[0m\n", + "\u001b[?25hDownloading numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.0 MB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.0/18.0 MB\u001b[0m \u001b[31m118.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m505.5/505.5 kB\u001b[0m \u001b[31m114.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m345.4/345.4 kB\u001b[0m \u001b[31m94.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: pytz, tzdata, numpy, pandas\n", + "Successfully installed numpy-1.26.4 pandas-2.2.1 pytz-2024.1 tzdata-2024.1\n" + ] + } + ], + "source": [ + "! pip install pandas" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['./.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2025Z_X1c9NiKwnV3CYmXI.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1950Z_Qk2aq2jark4HpBlG.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2040Z_Vmmj0B18ci4zTuy2.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1945Z_SsNqCRTyws7xrY3J.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1955Z_0JYT7c5H38fS0Jzy.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1950Z_yzePA9EprIlgYK2R.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2000Z_v34LxJHzVlXIw8cr.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T1955Z_et04WAteFcLN9YUu.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2000Z_yY6F9bSjH8QBAehk.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2000Z_PGu8Zq1rN9hTTm4k.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2000Z_zDdght9V3fwyY23H.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2010Z_GiGEsRw2KfolpslM.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2010Z_RJwgNb5JnrV3W9xv.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2015Z_SeCEZLsne6devXoH.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2020Z_t4m3yswisWjpr4o9.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2010Z_UaQ0DZP4OnkjJDzc.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2015Z_pDJdSu7mMtS0AIdw.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2025Z_PjbFu4ln25Zq2f3Z.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2030Z_YkevR3m6nigurLLv.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2035Z_HfhEvDRDJKxOx5MC.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2035Z_hJZ2XQfeaAiV60XT.json.gz',\n", + " './.data/AWSLogs/269733383066/CloudTrail/ap-northeast-1/2024/03/12/269733383066_CloudTrail_ap-northeast-1_20240312T2015Z_yOFUUoAdRpjUVbV0.json.gz']" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "import pandas as pd\n", + "import json\n", + "\n", + "datafiles = []\n", + "for root, dirs, files in os.walk(\"./.data\"):\n", + " for file in files:\n", + " if file.endswith(\".json.gz\"):\n", + " datafiles.append(os.path.join(root, file))\n", + "\n", + "datafiles" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " | eventVersion | \n", + "eventTime | \n", + "eventSource | \n", + "eventName | \n", + "awsRegion | \n", + "sourceIPAddress | \n", + "userAgent | \n", + "requestID | \n", + "eventID | \n", + "readOnly | \n", + "... | \n", + "requestParameters.DescribeVpcEndpointsRequest | \n", + "requestParameters.securityGroupIdSet.items | \n", + "requestParameters.vpcSet.items | \n", + "requestParameters.groupId | \n", + "requestParameters.dhcpOptionsId | \n", + "requestParameters.routeTableIdSet.items | \n", + "requestParameters.associationId | \n", + "requestParameters.networkAclIdSet.items | \n", + "requestParameters.subnetId | \n", + "requestParameters.vpcId | \n", + "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", + "1.08 | \n", + "2024-03-12T20:17:07Z | \n", + "sts.amazonaws.com | \n", + "AssumeRole | \n", + "ap-northeast-1 | \n", + "access-analyzer.amazonaws.com | \n", + "access-analyzer.amazonaws.com | \n", + "68977b77-a50e-48db-a53e-2beb90ff8661 | \n", + "b6ea10db-360e-3886-9d24-5764f2a8a98f | \n", + "True | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
1 | \n", + "1.09 | \n", + "2024-03-12T20:17:07Z | \n", + "s3.amazonaws.com | \n", + "GetBucketAcl | \n", + "ap-northeast-1 | \n", + "cloudtrail.amazonaws.com | \n", + "cloudtrail.amazonaws.com | \n", + "S29Z263Y106DP0YM | \n", + "f527b186-aa7a-46e0-b555-444960378d77 | \n", + "True | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
2 | \n", + "1.08 | \n", + "2024-03-12T20:17:09Z | \n", + "sts.amazonaws.com | \n", + "AssumeRole | \n", + "ap-northeast-1 | \n", + "access-analyzer.amazonaws.com | \n", + "access-analyzer.amazonaws.com | \n", + "4c7307cf-657a-4f65-8fcf-155f9b306b80 | \n", + "b8483928-1548-3b93-a9ef-a8f46aad3387 | \n", + "True | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
3 | \n", + "1.08 | \n", + "2024-03-12T20:17:11Z | \n", + "sts.amazonaws.com | \n", + "AssumeRole | \n", + "ap-northeast-1 | \n", + "access-analyzer.amazonaws.com | \n", + "access-analyzer.amazonaws.com | \n", + "790c1a5a-d727-4afa-8f3a-6c8568905644 | \n", + "1676c595-2c4b-386d-ae73-4f96eb555bb1 | \n", + "True | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
4 | \n", + "1.09 | \n", + "2024-03-12T20:17:11Z | \n", + "s3.amazonaws.com | \n", + "GetBucketAcl | \n", + "ap-northeast-1 | \n", + "cloudtrail.amazonaws.com | \n", + "cloudtrail.amazonaws.com | \n", + "DN585QA06J42CTZM | \n", + "7dfcc55f-6410-4ecc-b222-a026b7b65101 | \n", + "True | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
5 rows × 113 columns
\n", + "\n", + " | eventVersion | \n", + "eventTime | \n", + "eventSource | \n", + "eventName | \n", + "awsRegion | \n", + "sourceIPAddress | \n", + "userAgent | \n", + "requestID | \n", + "eventID | \n", + "readOnly | \n", + "... | \n", + "requestParameters.vpcSet.items | \n", + "requestParameters.groupId | \n", + "requestParameters.dhcpOptionsId | \n", + "requestParameters.routeTableIdSet.items | \n", + "requestParameters.associationId | \n", + "requestParameters.networkAclIdSet.items | \n", + "requestParameters.subnetId | \n", + "requestParameters.vpcId | \n", + "service | \n", + "api_call | \n", + "
---|
0 rows × 115 columns
\n", + "