Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CIAPP-5371] CI visibility: validate git tags #3100

Merged
merged 2 commits into from
Sep 4, 2023
Merged

[CIAPP-5371] CI visibility: validate git tags #3100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
534ddb0
CI visibility: validate DD_GIT_REPOSITORY_URL and log error if not pr…
anmarchenko Sep 1, 2023
d4711fd
CI visibility: validate git.commit.sha environment tag
anmarchenko Sep 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 40 additions & 4 deletions lib/datadog/ci/ext/environment.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ module CI
module Ext
# Defines constants for CI tags
module Environment
HEX_NUMBER_REGEXP = /[0-9a-f]{40}/i.freeze

TAG_JOB_NAME = 'ci.job.name'
TAG_JOB_URL = 'ci.job.url'
TAG_PIPELINE_ID = 'ci.pipeline.id'
Expand Down Expand Up @@ -71,7 +73,11 @@ def tags(env)
tags[key] ||= value
end

tags.reject { |_, v| v.nil? }
output = tags.reject { |_, v| v.nil? }

ensure_post_conditions(output)

output
end

def normalize_ref(name)
Expand Down Expand Up @@ -166,9 +172,7 @@ def extract_bitbucket(env)
pipeline_url = "https://bitbucket.org/#{env['BITBUCKET_REPO_FULL_NAME']}/addon/pipelines/home#" \
"!/results/#{env['BITBUCKET_BUILD_NUMBER']}"

repository_url = filter_sensitive_info(
env['BITBUCKET_GIT_SSH_ORIGIN'] || env['BITBUCKET_GIT_HTTP_ORIGIN']
)
repository_url = env['BITBUCKET_GIT_SSH_ORIGIN'] || env['BITBUCKET_GIT_HTTP_ORIGIN']

{
Core::Git::Ext::TAG_BRANCH => env['BITBUCKET_BRANCH'],
Expand Down Expand Up @@ -574,6 +578,38 @@ def extract_name_email(name_and_email)

[nil, name_and_email]
end

def ensure_post_conditions(tags)
validate_repository_url(tags[Core::Git::Ext::TAG_REPOSITORY_URL])
validate_git_sha(tags[Core::Git::Ext::TAG_COMMIT_SHA])
end

def validate_repository_url(repo_url)
return if !repo_url.nil? && !repo_url.empty?

Datadog.logger.error('DD_GIT_REPOSITORY_URL is not set or empty; no repo URL was automatically extracted')
end

def validate_git_sha(git_sha)
message = 'DD_GIT_COMMIT_SHA must be a full-length git SHA.'

if git_sha.nil? || git_sha.empty?
message += ' No value was set and no SHA was automatically extracted.'
Datadog.logger.error(message)
return
end

if git_sha.length < Core::Git::Ext::GIT_SHA_LENGTH
message += " Expected SHA length #{Core::Git::Ext::GIT_SHA_LENGTH}, was #{git_sha.length}."
Datadog.logger.error(message)
return
end

unless HEX_NUMBER_REGEXP =~ git_sha
message += " Expected SHA to be a valid HEX number, got #{git_sha}."
Datadog.logger.error(message)
end
end
end
end
end
Expand Down
2 changes: 2 additions & 0 deletions lib/datadog/core/git/ext.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ module Core
module Git
# Defines constants for Git tags
module Ext
GIT_SHA_LENGTH = 40

TAG_BRANCH = 'git.branch'
TAG_REPOSITORY_URL = 'git.repository_url'
TAG_TAG = 'git.tag'
Expand Down
2 changes: 2 additions & 0 deletions sig/datadog/core/git/ext.rbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ module Datadog
module Core
module Git
module Ext
GIT_SHA_LENGTH: ::Regexp

TAG_BRANCH: "git.branch"

TAG_REPOSITORY_URL: "git.repository_url"
Expand Down
165 changes: 131 additions & 34 deletions spec/datadog/ci/ext/environment_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@
RSpec.describe Datadog::CI::Ext::Environment do
FIXTURE_DIR = "#{File.dirname(__FILE__)}/fixtures/" # rubocop:disable all

let(:logger) { instance_double(Datadog::Core::Logger) }
before do
allow(Datadog).to receive(:logger).and_return(logger)
allow(logger).to receive(:debug)
allow(logger).to receive(:error)
end

describe '.tags' do
subject(:tags) do
ClimateControl.modify(environment_variables) { described_class.tags(env) }
Expand Down Expand Up @@ -110,49 +117,139 @@
include_context 'without git installed'

it 'does not fail' do
allow(Datadog.logger).to receive(:debug)

is_expected.to eq({})

expect(Datadog.logger).to have_received(:debug).with(/No such file or directory - git/).at_least(1).time
expect(logger).to have_received(:debug).with(/No such file or directory - git/).at_least(1).time
end
end

context 'user provided metadata' do
include_context 'with git fixture', 'gitdir_with_commit'
let(:env) do
{
'DD_GIT_REPOSITORY_URL' => 'https://datadoghq.com/git/user-provided.git',
'DD_GIT_COMMIT_SHA' => '9322ca1d57975b49b8c00b449d21b06660ce8b5c',
'DD_GIT_BRANCH' => 'my-branch',
'DD_GIT_TAG' => 'my-tag',
'DD_GIT_COMMIT_MESSAGE' => 'provided message',
'DD_GIT_COMMIT_AUTHOR_NAME' => 'user',
'DD_GIT_COMMIT_AUTHOR_EMAIL' => 'user@provided.com',
'DD_GIT_COMMIT_AUTHOR_DATE' => '2021-06-18T18:35:10+00:00',
'DD_GIT_COMMIT_COMMITTER_NAME' => 'user committer',
'DD_GIT_COMMIT_COMMITTER_EMAIL' => 'user-committer@provided.com',
'DD_GIT_COMMIT_COMMITTER_DATE' => '2021-06-19T18:35:10+00:00',
}
end
context 'when required values are present' do
include_context 'with git fixture', 'gitdir_with_commit'

it 'returns user provided metadata' do
is_expected.to eq(
let(:env) do
{
'ci.workspace_path' => "#{Dir.pwd}/spec/datadog/ci/ext/fixtures/git",
'git.branch' => env['DD_GIT_BRANCH'],
'git.tag' => env['DD_GIT_TAG'],
'git.commit.author.date' => env['DD_GIT_COMMIT_AUTHOR_DATE'],
'git.commit.author.email' => env['DD_GIT_COMMIT_AUTHOR_EMAIL'],
'git.commit.author.name' => env['DD_GIT_COMMIT_AUTHOR_NAME'],
'git.commit.committer.date' => env['DD_GIT_COMMIT_COMMITTER_DATE'],
'git.commit.committer.email' => env['DD_GIT_COMMIT_COMMITTER_EMAIL'],
'git.commit.committer.name' => env['DD_GIT_COMMIT_COMMITTER_NAME'],
'git.commit.message' => env['DD_GIT_COMMIT_MESSAGE'],
'git.commit.sha' => env['DD_GIT_COMMIT_SHA'],
'git.repository_url' => env['DD_GIT_REPOSITORY_URL']
'DD_GIT_REPOSITORY_URL' => 'https://datadoghq.com/git/user-provided.git',
'DD_GIT_COMMIT_SHA' => '9322CA1d57975b49b8c00b449d21b06660ce8b5c',
'DD_GIT_BRANCH' => 'my-branch',
'DD_GIT_TAG' => 'my-tag',
'DD_GIT_COMMIT_MESSAGE' => 'provided message',
'DD_GIT_COMMIT_AUTHOR_NAME' => 'user',
'DD_GIT_COMMIT_AUTHOR_EMAIL' => 'user@provided.com',
'DD_GIT_COMMIT_AUTHOR_DATE' => '2021-06-18T18:35:10+00:00',
'DD_GIT_COMMIT_COMMITTER_NAME' => 'user committer',
'DD_GIT_COMMIT_COMMITTER_EMAIL' => 'user-committer@provided.com',
'DD_GIT_COMMIT_COMMITTER_DATE' => '2021-06-19T18:35:10+00:00',
}
)
end

it 'returns user provided metadata' do
is_expected.to eq(
{
'ci.workspace_path' => "#{Dir.pwd}/spec/datadog/ci/ext/fixtures/git",
'git.branch' => env['DD_GIT_BRANCH'],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we change the assertion with the actual value instead of an indirection through environment variable?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sorry, I missed your comment :(

I think in this case though it would make it only more magical to put an actual value here as these env vars are defined in the same context

'git.tag' => env['DD_GIT_TAG'],
'git.commit.author.date' => env['DD_GIT_COMMIT_AUTHOR_DATE'],
'git.commit.author.email' => env['DD_GIT_COMMIT_AUTHOR_EMAIL'],
'git.commit.author.name' => env['DD_GIT_COMMIT_AUTHOR_NAME'],
'git.commit.committer.date' => env['DD_GIT_COMMIT_COMMITTER_DATE'],
'git.commit.committer.email' => env['DD_GIT_COMMIT_COMMITTER_EMAIL'],
'git.commit.committer.name' => env['DD_GIT_COMMIT_COMMITTER_NAME'],
'git.commit.message' => env['DD_GIT_COMMIT_MESSAGE'],
'git.commit.sha' => env['DD_GIT_COMMIT_SHA'],
'git.repository_url' => env['DD_GIT_REPOSITORY_URL']
}
)
end
end

context 'with no git information extracted' do
include_context 'without git installed'

context 'when DD_GIT_REPOSITORY_URL is missing' do
let(:env) do
{
'DD_GIT_COMMIT_SHA' => '9322ca1d57975b49b8c00b449d21b06660ce8b5c',
}
end

it 'logs an error' do
is_expected.to eq(
{
'git.commit.sha' => env['DD_GIT_COMMIT_SHA'],
}
)

expect(logger).to have_received(:error).with(
'DD_GIT_REPOSITORY_URL is not set or empty; no repo URL was automatically extracted'
)
end
end

context 'when DD_GIT_COMMIT_SHA is missing' do
let(:env) do
{
'DD_GIT_REPOSITORY_URL' => 'https://datadoghq.com/git/user-provided.git',
}
end

it 'logs an error' do
is_expected.to eq(
{
'git.repository_url' => env['DD_GIT_REPOSITORY_URL']
}
)

expect(logger).to have_received(:error).with(
'DD_GIT_COMMIT_SHA must be a full-length git SHA. No value was set and no SHA was automatically extracted.'
)
end
end

context 'when DD_GIT_COMMIT_SHA has invalid length' do
let(:env) do
{
'DD_GIT_COMMIT_SHA' => '9322ca1d57975b49b8c00b449d21b06660ce8b5',
'DD_GIT_REPOSITORY_URL' => 'https://datadoghq.com/git/user-provided.git',
}
end

it 'logs an error' do
is_expected.to eq(
{
'git.commit.sha' => env['DD_GIT_COMMIT_SHA'],
'git.repository_url' => env['DD_GIT_REPOSITORY_URL']
}
)

expect(logger).to have_received(:error).with(
'DD_GIT_COMMIT_SHA must be a full-length git SHA. Expected SHA length 40, was 39.'
)
end
end

context 'when DD_GIT_COMMIT_SHA is not a valid hex number' do
let(:env) do
{
'DD_GIT_COMMIT_SHA' => '9322ca1d57975by9b8c00b449d21b06660ce8b5c',
'DD_GIT_REPOSITORY_URL' => 'https://datadoghq.com/git/user-provided.git',
}
end

it 'logs an error' do
is_expected.to eq(
{
'git.commit.sha' => env['DD_GIT_COMMIT_SHA'],
'git.repository_url' => env['DD_GIT_REPOSITORY_URL']
}
)

expect(logger).to have_received(:error).with(
'DD_GIT_COMMIT_SHA must be a full-length git SHA. ' \
'Expected SHA to be a valid HEX number, got 9322ca1d57975by9b8c00b449d21b06660ce8b5c.'
)
end
end
end
end
end
Expand Down