Skip to content

Commit

Permalink
MOTOR-1360 Use AssumeRole for S3 Access in Evergreen Builds (#303)
Browse files Browse the repository at this point in the history
  • Loading branch information
blink1073 authored Sep 3, 2024
1 parent 48f34cb commit b652685
Showing 1 changed file with 13 additions and 169 deletions.
182 changes: 13 additions & 169 deletions .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,21 +54,18 @@ functions:
export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
export UPLOAD_BUCKET="${project}"
cat <<EOT > expansion.yml
CURRENT_VERSION: "$CURRENT_VERSION"
DRIVERS_TOOLS: "$DRIVERS_TOOLS"
MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME"
MONGODB_BINARIES: "$MONGODB_BINARIES"
UPLOAD_BUCKET: "$UPLOAD_BUCKET"
PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
PREPARE_SHELL: |
set -o errexit
export DRIVERS_TOOLS="$DRIVERS_TOOLS"
export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
export MONGODB_BINARIES="$MONGODB_BINARIES"
export UPLOAD_BUCKET="$UPLOAD_BUCKET"
export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
export PATH="$MONGODB_BINARIES:$PATH"
Expand Down Expand Up @@ -99,58 +96,6 @@ functions:
fi
echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config
"upload release":
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: ${project}.tar.gz
remote_file: ${UPLOAD_BUCKET}/${project}-${CURRENT_VERSION}.tar.gz
bucket: mciuploads
permissions: public-read
content_type: ${content_type|application/x-gzip}

# Upload build artifacts that other tasks may depend on
# Note this URL needs to be totally unique, while predictable for the next task
# so it can automatically download the artifacts
"upload build":
# Compress and upload the entire build directory
- command: archive.targz_pack
params:
# Example: mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz
target: "${build_id}.tar.gz"
source_dir: ${PROJECT_DIRECTORY}/
include:
- "./**"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: ${build_id}.tar.gz
# Example: /mciuploads/${UPLOAD_BUCKET}/gcc49/9dfb7d741efbca16faa7859b9349d7a942273e43/debug-compile-nosasl-nossl/mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${task_name}/${build_id}.tar.gz
bucket: mciuploads
permissions: public-read
content_type: ${content_type|application/x-gzip}

"fetch build":
- command: shell.exec
params:
continue_on_err: true
script: "set -o xtrace && rm -rf ${PROJECT_DIRECTORY}"
- command: s3.get
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${BUILD_NAME}/${build_id}.tar.gz
bucket: mciuploads
local_file: build.tar.gz
- command: shell.exec
params:
continue_on_err: true
# EVG-1105: Use s3.get extract_to: ./
script: "set -o xtrace && cd .. && rm -rf ${PROJECT_DIRECTORY} && mkdir ${PROJECT_DIRECTORY}/ && tar xf build.tar.gz -C ${PROJECT_DIRECTORY}/"

"exec compile script" :
- command: shell.exec
type: test
Expand All @@ -169,137 +114,38 @@ functions:
${PREPARE_SHELL}
[ -f ${PROJECT_DIRECTORY}/${file} ] && sh ${PROJECT_DIRECTORY}/${file} || echo "${PROJECT_DIRECTORY}/${file} not available, skipping"
"upload docs" :
- command: shell.exec
params:
silent: true
script: |
export AWS_ACCESS_KEY_ID=${aws_key}
export AWS_SECRET_ACCESS_KEY=${aws_secret}
aws s3 cp ${PROJECT_DIRECTORY}/doc/html s3://mciuploads/${UPLOAD_BUCKET}/docs/${CURRENT_VERSION} --recursive --acl public-read --region us-east-1
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: ${PROJECT_DIRECTORY}/doc/html/index.html
remote_file: ${UPLOAD_BUCKET}/docs/${CURRENT_VERSION}/index.html
bucket: mciuploads
permissions: public-read
content_type: text/html
display_name: "Rendered docs"

"upload coverage" :
- command: shell.exec
params:
silent: true
script: |
export AWS_ACCESS_KEY_ID=${aws_key}
export AWS_SECRET_ACCESS_KEY=${aws_secret}
aws s3 cp ${PROJECT_DIRECTORY}/coverage s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/ --recursive --acl public-read --region us-east-1
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: ${PROJECT_DIRECTORY}/coverage/index.html
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/index.html
bucket: mciuploads
permissions: public-read
content_type: text/html
display_name: "Coverage Report"

"upload scan artifacts" :
- command: shell.exec
type: test
params:
script: |
cd
if find ${PROJECT_DIRECTORY}/scan -name \*.html | grep -q html; then
(cd ${PROJECT_DIRECTORY}/scan && find . -name index.html -exec echo "<li><a href='{}'>{}</a></li>" \;) >> scan.html
else
echo "No issues found" > scan.html
fi
- command: shell.exec
params:
silent: true
script: |
export AWS_ACCESS_KEY_ID=${aws_key}
export AWS_SECRET_ACCESS_KEY=${aws_secret}
aws s3 cp ${PROJECT_DIRECTORY}/scan s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/ --recursive --acl public-read --region us-east-1
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: ${PROJECT_DIRECTORY}/scan.html
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/index.html
bucket: mciuploads
permissions: public-read
content_type: text/html
display_name: "Scan Build Report"

"upload mo artifacts":
- command: ec2.assume_role
params:
role_arn: ${assume_role_arn}
- command: shell.exec
params:
script: |
${PREPARE_SHELL}
find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf mongodb-logs.tar.gz
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: mongodb-logs.tar.gz
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz
bucket: mciuploads
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz
bucket: ${aws_bucket}
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "mongodb-logs.tar.gz"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
aws_key: ${AWS_ACCESS_KEY_ID}
aws_secret: ${AWS_SECRET_ACCESS_KEY}
aws_session_token: ${AWS_SESSION_TOKEN}
local_file: ${DRIVERS_TOOLS}/.evergreen/orchestration/server.log
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log
bucket: mciuploads
remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log
bucket: ${aws_bucket}
permissions: public-read
content_type: ${content_type|text/plain}
display_name: "orchestration.log"

"upload working dir":
- command: archive.targz_pack
params:
target: "working-dir.tar.gz"
source_dir: ${PROJECT_DIRECTORY}/
include:
- "./**"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: working-dir.tar.gz
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz
bucket: mciuploads
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "working-dir.tar.gz"
- command: archive.targz_pack
params:
target: "drivers-dir.tar.gz"
source_dir: ${DRIVERS_TOOLS}
include:
- "./**"
exclude_files:
# Windows cannot read the mongod *.lock files because they are locked.
- "*.lock"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: drivers-dir.tar.gz
remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz
bucket: mciuploads
permissions: public-read
content_type: ${content_type|application/x-gzip}
display_name: "drivers-dir.tar.gz"

"upload test results":
- command: attach.results
params:
Expand Down Expand Up @@ -462,8 +308,6 @@ pre:
- func: "install dependencies"

post:
# Disabled, causing timeouts
# - func: "upload working dir"
- func: "upload mo artifacts"
- func: "upload test results"
- func: "stop mongo-orchestration"
Expand Down

0 comments on commit b652685

Please sign in to comment.