Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: save test artifacts for fail tests #1501

Merged
merged 1 commit into from
Feb 19, 2024
Merged

ci: save test artifacts for fail tests #1501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/actions/s3cmd/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,10 @@ runs:
exit 1
;;
esac

echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
echo "S3_TEST_ARTIFACTS_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
env:
s3_key_id: ${{ inputs.s3_key_id }}
s3_secret_access_key: ${{ inputs.s3_key_secret }}
11 changes: 7 additions & 4 deletions .github/actions/test_ya/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ runs:
echo "LOG_DIR=$TMP_DIR/logs" >> $GITHUB_ENV
echo "OUT_DIR=$TMP_DIR/out" >> $GITHUB_ENV
echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" >> $GITHUB_ENV
echo "TEST_ARTIFACTS_DIR=$TMP_DIR/test_artifacts" >> $GITHUB_ENV
echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports" >> $GITHUB_ENV
echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml" >> $GITHUB_ENV
echo "JUNIT_REPORT_PARTS=$TMP_DIR/junit-split" >> $GITHUB_ENV
Expand All @@ -73,7 +74,7 @@ runs:
shell: bash
run: |
rm -rf $TMP_DIR $JUNIT_REPORT_XML $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $TEST_ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR

- name: Install Node required for Testmo CLI
uses: actions/setup-node@v3
Expand Down Expand Up @@ -222,7 +223,7 @@ runs:
./ya test "${params[@]}" \
--bazel-remote-put --bazel-remote-username "${{ inputs.bazel_remote_username }}" --bazel-remote-password "${{ inputs.bazel_remote_password }}" -DCONSISTENT_DEBUG \
--log-file "$LOG_DIR/ya_log_prewarm.txt" --evlog-file "$LOG_DIR/ya_evlog_prewarm.jsonl" \
--dist-cache-evict-bins --cache-tests --no-dir-outputs --test-node-output-limit 100000 --drop-graph-result-before-tests || (
--dist-cache-evict-bins --cache-tests --no-dir-outputs --test-node-output-limit 1000000 --drop-graph-result-before-tests || (
RC=$?
echo "::debug::ya test RC=$RC"
)
Expand All @@ -231,7 +232,7 @@ runs:
echo "::debug::save tests reports"
./ya test "${params[@]}" \
--stat --log-file "$LOG_DIR/ya_log.txt" --evlog-file "$LOG_DIR/ya_evlog.jsonl" -DCONSISTENT_DEBUG \
--cache-tests --dist-cache-evict-bins --no-dir-outputs --test-node-output-limit 100000 --drop-graph-result-before-tests \
--cache-tests --dist-cache-evict-bins --no-dir-outputs --test-node-output-limit 1000000 --drop-graph-result-before-tests \
--junit "$JUNIT_REPORT_XML" --output "$OUT_DIR" || (
RC=$?
if [ $RC -ne 0 ]; then
Expand Down Expand Up @@ -259,7 +260,8 @@ runs:
-m .github/config/muted_ya.txt \
--ya-out "$OUT_DIR" \
--log-url-prefix "$S3_URL_PREFIX/logs/" \
--log-out-dir "$ARTIFACTS_DIR/logs/" \
--test-stuff-out "$TEST_ARTIFACTS_DIR/" \
--test-stuff-prefix "$S3_TEST_ARTIFACTS_URL_PREFIX/" \
"$JUNIT_REPORT_XML"

.github/scripts/tests/split-junit.py -o "$JUNIT_REPORT_PARTS" "$JUNIT_REPORT_XML"
Expand Down Expand Up @@ -321,6 +323,7 @@ runs:
run: |
echo "::group::s3-sync"
s3cmd sync -r --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$ARTIFACTS_DIR/" "$S3_BUCKET_PATH/"
s3cmd sync -r --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$TEST_ARTIFACTS_DIR/" "$S3_TEST_ARTIFACTS_BUCKET_PATH/"
echo "::endgroup::"

- name: sync logs results to s3
Expand Down
49 changes: 47 additions & 2 deletions .github/scripts/tests/transform-ya-junit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import os
import sys
import urllib.parse
import zipfile
from xml.etree import ElementTree as ET
from mute_utils import mute_target, pattern_to_re
from junit_utils import add_junit_link_property, is_faulty_testcase
Expand Down Expand Up @@ -53,6 +54,10 @@ class YTestReportTrace:
def __init__(self, out_root):
self.out_root = out_root
self.traces = {}
self.logs_dir = None

def abs_path(self, path):
return path.replace("$(BUILD_ROOT)", self.out_root)

def load(self, subdir):
test_results_dir = os.path.join(self.out_root, f"{subdir}/test-results/")
Expand All @@ -61,6 +66,7 @@ def load(self, subdir):
log_print(f"Directory {test_results_dir} doesn't exist")
return

# find the test result
for folder in os.listdir(test_results_dir):
fn = os.path.join(self.out_root, test_results_dir, folder, "ytest.report.trace")

Expand All @@ -76,6 +82,9 @@ def load(self, subdir):
subtest = event["subtest"]
cls = cls.replace("::", ".")
self.traces[(cls, subtest)] = event
logs_dir = self.abs_path(event['logs']['logsdir'])
self.logs_dir = logs_dir
break

def has(self, cls, name):
return (cls, name) in self.traces
Expand All @@ -93,7 +102,7 @@ def get_logs(self, cls, name):
if k == "logsdir":
continue

result[k] = path.replace("$(BUILD_ROOT)", self.out_root)
result[k] = self.abs_path(path)

return result

Expand Down Expand Up @@ -135,7 +144,26 @@ def save_log(build_root, fn, out_dir, log_url_prefix, trunc_size):
return f"{log_url_prefix}{quoted_fpath}"


def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_prefix, log_out_dir, log_trunc_size):
def save_zip(suite_name, out_dir, url_prefix, logs_dir):
arc_name = f"{suite_name.replace('/', '-')}.zip"

arc_fn = os.path.join(out_dir, arc_name)

zf = zipfile.ZipFile(arc_fn, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=9)

log_print(f"put {logs_dir} into {arc_name}")
for root, dirs, files in os.walk(logs_dir):
for f in files:
filename = os.path.join(root, f)
zf.write(filename, os.path.relpath(filename, logs_dir))
zf.close()

quoted_fpath = urllib.parse.quote(arc_name)
return f"{url_prefix}{quoted_fpath}"


def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_prefix, log_out_dir, log_trunc_size,
test_stuff_out, test_stuff_prefix):
tree = ET.parse(fp)
root = tree.getroot()

Expand All @@ -144,11 +172,14 @@ def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_pre
traces = YTestReportTrace(ya_out_dir)
traces.load(suite_name)

has_fail_tests = False

for case in suite.findall("testcase"):
test_name = case.get("name")
case.set("classname", suite_name)

is_fail = is_faulty_testcase(case)
has_fail_tests |= is_fail

if mute_check(suite_name, test_name):
log_print("mute", suite_name, test_name)
Expand All @@ -164,6 +195,16 @@ def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_pre
url = save_log(ya_out_dir, fn, log_out_dir, log_url_prefix, log_trunc_size)
add_junit_link_property(case, name, url)

if has_fail_tests:
if not traces.logs_dir:
log_print(f"no logsdir for {suite_name}")
continue

url = save_zip(suite_name, test_stuff_out, test_stuff_prefix, traces.logs_dir)

for case in suite.findall("testcase"):
add_junit_link_property(case, 'logsdir', url)

if save_inplace:
tree.write(fp.name)
else:
Expand All @@ -187,6 +228,8 @@ def main():
help="truncate log after specific size, 0 disables truncation",
)
parser.add_argument("--ya-out", help="ya make output dir (for searching logs and artifacts)")
parser.add_argument('--test-stuff-out', help='output folder for archive testing_out_stuff')
parser.add_argument('--test-stuff-prefix', help='url prefix for testing_out_stuff')
parser.add_argument("in_file", type=argparse.FileType("r"))

args = parser.parse_args()
Expand All @@ -204,6 +247,8 @@ def main():
args.log_url_prefix,
args.log_out_dir,
args.log_trunc_size,
args.test_stuff_out,
args.test_stuff_prefix,
)


Expand Down
Loading