Skip to content

Commit db5a3b9

Browse files
EgorkaZAlnenUgnineSirdis
authored
YQ-3035: FQ gather stable 24-1 lost ones (#3651)
Co-authored-by: Alexey Uzhegov <auzhegov@ydb.tech> Co-authored-by: Vasily Gerasimov <UgnineSirdis@ydb.tech>
1 parent ce30b98 commit db5a3b9

30 files changed

+820
-152
lines changed

‎.github/actions/build/action.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ inputs:
1111
required: false
1212
default: ""
1313
description: "extra compile flags will be added to the end of C_FLAGS and CXX_FLAGS"
14+
ninja_target:
15+
required: false
16+
type: string
1417

1518
runs:
1619
using: "composite"
@@ -59,7 +62,7 @@ runs:
5962
export CCACHE_SLOPPINESS=locale
6063
export CCACHE_MAXSIZE=50G
6164
cd ../build
62-
ninja
65+
ninja ${{ inputs.ninja_target }}
6366
ccache -s
6467
df -h
6568
- name: report Build failed

‎.github/actions/build_and_test_ya/action.yml‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
name: Ya-Build-and-Test
2+
inputs:
3+
build_target:
4+
type: string
5+
default: "ydb/"
6+
description: "limit build and test to specific target"
7+
build_preset:
8+
type: string
9+
run_build:
10+
type: boolean
11+
default: true
12+
description: "run build"
13+
run_tests:
14+
type: boolean
15+
default: true
16+
description: "run tests"
17+
run_tests_if_build_fails:
18+
default: "true"
19+
description: "run tests if build fails"
20+
test_threads:
21+
type: string
22+
default: 28
23+
description: "Test threads count"
24+
link_threads:
25+
type: string
26+
default: 8
27+
description: "link threads count"
28+
test_size:
29+
type: string
30+
default: "small,medium,large"
31+
test_type:
32+
type: string
33+
default: "unittest,py3test,py2test,pytest"
34+
increment:
35+
type: boolean
36+
required: true
37+
description: If true, compares build graphs between the current and previous commits to find a list of test suites to run. Otherwise, runs all tests.
38+
folder_prefix:
39+
type: string
40+
default: "ya-"
41+
put_build_results_to_cache:
42+
type: boolean
43+
default: true
44+
secs:
45+
type: string
46+
default: ""
47+
vars:
48+
type: string
49+
default: ""
50+
defaults:
51+
run:
52+
shell: bash
53+
runs:
54+
using: "composite"
55+
steps:
56+
- name: comment-build-start
57+
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
58+
shell: bash
59+
env:
60+
BUILD_PRESET: ${{ inputs.build_preset }}
61+
GITHUB_TOKEN: ${{ github.token }}
62+
run: |
63+
jobs_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs"
64+
# tricky: we are searching job with name that contains build_preset
65+
check_url=$(curl -s $jobs_url | jq --arg n "$BUILD_PRESET" -r '.jobs[] | select(.name | contains($n)) | .html_url')
66+
67+
echo "Pre-commit [check]($check_url) for $(git rev-parse HEAD) has started." | .github/scripts/tests/comment-pr.py --rewrite
68+
69+
- name: Prepare s3cmd
70+
uses: ./.github/actions/s3cmd
71+
with:
72+
s3_bucket: ${{ fromJSON( inputs.vars ).AWS_BUCKET }}
73+
s3_endpoint: ${{ fromJSON( inputs.vars ).AWS_ENDPOINT }}
74+
s3_key_id: ${{ fromJSON( inputs.secs ).AWS_KEY_ID }}
75+
s3_key_secret: ${{ fromJSON( inputs.secs ).AWS_KEY_VALUE }}
76+
folder_prefix: ya-
77+
build_preset: ${{ inputs.build_preset }}
78+
79+
- name: Build
80+
uses: ./.github/actions/build_ya
81+
id: build
82+
if: ${{ inputs.run_build == 'true' }}
83+
with:
84+
build_target: ${{ inputs.build_target }}
85+
build_preset: ${{ inputs.build_preset }}
86+
bazel_remote_uri: ${{ fromJSON( inputs.vars ).REMOTE_CACHE_URL || '' }}
87+
bazel_remote_username: ${{ inputs.put_build_results_to_cache && fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
88+
bazel_remote_password: ${{ inputs.put_build_results_to_cache && fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
89+
link_threads: ${{ inputs.link_threads }}
90+
91+
- name: Generate ya.make with affected test suites list
92+
if: inputs.run_tests == 'true' && inputs.increment == 'true'
93+
uses: ./.github/actions/graph_compare
94+
95+
- name: Check if there's a list of tests to run
96+
id: test_run_choice
97+
shell: bash
98+
run: |
99+
if [ -f ya.make ];then
100+
echo "target='.'" >> $GITHUB_OUTPUT
101+
echo "Listed test targets: "
102+
cat ya.make
103+
else
104+
echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
105+
fi
106+
107+
- name: Run tests
108+
uses: ./.github/actions/test_ya
109+
if: ${{ inputs.run_tests == 'true' && (steps.build.outputs.success == 'true' || inputs.run_tests_if_build_fails == 'true') }}
110+
with:
111+
build_target: ${{ steps.test_run_choice.outputs.target }}
112+
build_preset: ${{ inputs.build_preset }}
113+
test_size: ${{ inputs.test_size }}
114+
testman_token: ${{ fromJSON( inputs.secs ).TESTMO_TOKEN }}
115+
testman_url: ${{ fromJSON( inputs.vars ).TESTMO_URL }}
116+
testman_project_id: ${{ fromJSON( inputs.vars ).TESTMO_PROJECT_ID }}
117+
link_threads: ${{ inputs.link_threads }}
118+
test_threads: ${{ inputs.test_threads }}
119+
120+
- name: Notify about failed build
121+
if: ${{ steps.build.outputs.success != 'true' && inputs.run_tests == 'true' && inputs.run_tests_if_build_fails == 'false' }}
122+
shell: bash
123+
run: |
124+
echo 'Build failed. See the [build log](${{ steps.build.outputs.log_url }}).' >> $GITHUB_STEP_SUMMARY
125+
126+
if [[ "$GITHUB_EVENT_NAME" =~ ^pull_request ]]; then
127+
echo "Tests run skipped." | .github/scripts/tests/comment-pr.py --fail
128+
fi
129+
130+
exit 1
131+
132+
- name: comment-if-cancel
133+
shell: bash
134+
if: cancelled() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target')
135+
env:
136+
BUILD_PRESET: ${{ inputs.build_preset }}
137+
GITHUB_TOKEN: ${{ github.token }}
138+
run: echo "Check cancelled" | .github/scripts/tests/comment-pr.py --color black

‎.github/actions/build_ya/action.yml‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ inputs:
77
build_preset:
88
required: true
99
default: "relwithdebinfo"
10-
description: "relwithdebinfo, release-asan, release-tsan"
10+
description: "debug, relwithdebinfo, release-asan, release-tsan, release, release-cmake14"
1111
bazel_remote_uri:
1212
required: false
1313
description: "bazel-remote endpoint"
@@ -21,7 +21,13 @@ inputs:
2121
required: false
2222
default: "8"
2323
description: "link threads count"
24-
24+
outputs:
25+
success:
26+
value: ${{ steps.build.outputs.status }}
27+
description: "build success"
28+
log_url:
29+
value: ${{ steps.init.outputs.log_url }}
30+
description: "build log url"
2531
runs:
2632
using: "composite"
2733
steps:
@@ -34,10 +40,15 @@ runs:
3440
echo "SHELLOPTS=xtrace" >> $GITHUB_ENV
3541
export TMP_DIR=$(pwd)/tmp_build
3642
echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
43+
44+
export log_url="$S3_URL_PREFIX/build_logs/ya_make.log"
45+
3746
rm -rf $TMP_DIR && mkdir $TMP_DIR
3847
3948
echo "BUILD_PRESET=$build_preset" >> $GITHUB_ENV
4049
echo "GITHUB_TOKEN=${{ github.token }}" >> $GITHUB_ENV
50+
echo "LOG_URL=$log_url" >> $GITHUB_ENV
51+
echo "log_url=$log_url" >> $GITHUB_OUTPUT
4152
4253
- name: build
4354
id: build
@@ -67,6 +78,14 @@ runs:
6778
relwithdebinfo)
6879
build_type=relwithdebinfo
6980
;;
81+
release)
82+
build_type=release
83+
;;
84+
release-clang14)
85+
build_type=release
86+
extra_params+=(--target-platform="CLANG14-LINUX-X86_64")
87+
extra_params+=(-DLLD_VERSION=16)
88+
;;
7089
release-asan)
7190
build_type=release
7291
extra_params+=(--sanitize="address")
@@ -99,7 +118,7 @@ runs:
99118
./ya make -k --build "${build_type}" --force-build-depends -D'BUILD_LANGUAGES=CPP PY3 PY2 GO' -T --stat -DCONSISTENT_DEBUG \
100119
--log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \
101120
--cache-size 512G --link-threads "${{ inputs.link_threads }}" \
102-
"${extra_params[@]}" |& tee $TMP_DIR/ya_make.log || (
121+
"${extra_params[@]}" |& tee $TMP_DIR/ya_make.log && echo "status=true" >> $GITHUB_OUTPUT || (
103122
RC=$?
104123
echo "::debug::ya make RC=$RC"
105124
echo "status=failed" >> $GITHUB_OUTPUT
@@ -118,10 +137,8 @@ runs:
118137
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
119138
shell: bash
120139
run: |
121-
log_url="$S3_URL_PREFIX/build_logs/ya_make.log"
122-
123140
if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
124-
echo "Build failed. see the [build logs]($log_url)." | .github/scripts/tests/comment-pr.py --fail
141+
echo "Build failed. see the [build logs]($LOG_URL)." | .github/scripts/tests/comment-pr.py --fail
125142
else
126143
echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
127144
fi

‎.github/actions/graph_compare/action.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: graph_compare
2+
description: Compare graphs between current and previous commits (merge commit base in case of a merge commit), and list affected tests in ya.make
3+
runs:
4+
using: "composite"
5+
steps:
6+
- name: original_ref
7+
id: oref
8+
shell: bash
9+
run: |
10+
echo "value=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
11+
- name: generate_ya_make
12+
shell: bash
13+
run: |
14+
./.github/scripts/graph_compare.sh ${{ steps.oref.outputs.value }}~1 ${{ steps.oref.outputs.value }}
15+
- name: restore_ref
16+
shell: bash
17+
run: |
18+
git checkout ${{ steps.oref.outputs.value }}

‎.github/actions/s3cmd/action.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,10 @@ runs:
5454
exit 1
5555
;;
5656
esac
57-
5857
echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
5958
echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
59+
echo "S3_TEST_ARTIFACTS_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
60+
echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
6061
env:
6162
s3_key_id: ${{ inputs.s3_key_id }}
6263
s3_secret_access_key: ${{ inputs.s3_key_secret }}

0 commit comments

Comments
 (0)