-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(triton-linalg): add ci for triton-linalg
- Loading branch information
Showing
17 changed files
with
1,363 additions
and
198 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| import time | ||
| import sys | ||
| import os | ||
| import argparse | ||
| ''' | ||
| Get the result information fed back from the job server. If it is 'success' or 'failed', exit the pipeline. Otherwise, continue to monitor job information every 2 seconds. | ||
| output_path: the target file that you want to combine sub log with. | ||
| list_path: the list of sub log name. When it is updated, the correspondding file will be add to output tail. | ||
| list_dir_path: the dir path where sub logs stored. | ||
| status_path: the path of status file. When status file is written to "success" or "fail", exit script. | ||
| ''' | ||
|
|
||
| def combine_log(output_path, list_path, list_dir_path, status_path): | ||
| # list_pos stores the last position that pointer of list file pointed to. | ||
| list_pos = 0 | ||
| while True: | ||
| list_file = open(list_path, 'r') | ||
| list_file.seek(list_pos) | ||
| # read all lines starting from list_pos. | ||
| items = list_file.readlines() | ||
| # update list_pos | ||
| list_pos = list_file.tell() | ||
| # if read any line | ||
| if items is not None: | ||
| items.sort() | ||
| for item in items: | ||
| sub_path = item.strip() | ||
| if sub_path != "": | ||
| file_name = list_dir_path + '/' + sub_path | ||
| # while True: | ||
| if os.path.exists(file_name): | ||
| os.system('cat ' + file_name + ' >> ' + output_path) | ||
| # break | ||
| # check status_file, when read "success" or "fail" exit cycle, or else, sleep some seconds and start from beginning. | ||
| status_file = open(status_path) | ||
| status = status_file.readline().strip() | ||
| status_file.close() | ||
| if "fail" in status or "success" in status or "Success" in status or "Fail" in status or "error" in status or "Error" in status: | ||
| break | ||
| else: | ||
| time.sleep(2) | ||
|
|
||
| if __name__ == '__main__': | ||
| parser = argparse.ArgumentParser(description="Monitor and concatenate files based on a list.") | ||
| parser.add_argument('output_path', type=str, help='The path to the output file.') | ||
| parser.add_argument('list_path', type=str, help='The path to the list file containing sub-paths.') | ||
| parser.add_argument('list_dir_path', type=str, help='The base directory where sub-paths are located.') | ||
| parser.add_argument('status_path', type=str, help='The path to the status file.') | ||
|
|
||
| args = parser.parse_args() | ||
| combine_log(args.output_path, args.list_path, args.list_dir_path, args.status_path) | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| import time | ||
| import sys | ||
| import os | ||
|
|
||
| def file_guard(): | ||
| # where stores the last position that pointer pointed to. | ||
| where= 0 | ||
| while True: | ||
| file = open(guard_log_file, "r") | ||
| file.seek(where) | ||
| # if read any lines, call system echo to print each line. | ||
| for line in file.readlines(): | ||
| new_line = line.strip().replace("\'", "_").replace("\"", "_") | ||
| os.system('echo ' + "'%s'" % new_line) | ||
| # update where | ||
| where = file.tell() | ||
| file.close() | ||
| # check status, end process when read "success" or "fail" | ||
| status_file = open(guard_status_file, "r") | ||
| line = status_file.readline().strip() | ||
| status_file.close() | ||
| if "success" in line or "Success" in line: | ||
| print("Task success.") | ||
| break | ||
| elif "fail" in line or "Fail" in line or "error" in line or "Error" in line: | ||
| print("Task Fail.") | ||
| exit(-1) | ||
| # sleep for a while | ||
| time.sleep(2) | ||
| if __name__ == '__main__': | ||
| parser = argparse.ArgumentParser(description="Monitor a log file and echo lines, check status to stop.") | ||
| parser.add_argument('guard_status_file', type=str, help='The path to the status file.') | ||
| parser.add_argument('guard_log_file', type=str, help='The path to the log file.') | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| file_guard(args.guard_status_file, args.guard_log_file) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| # /bin/bash | ||
| # Get PR id | ||
| PR_string=$(echo $GITHUB_REF | grep -Eo "/[0-9]*/") | ||
| pr_id=(${PR_string//// }) | ||
|
|
||
| # Generate time stamp | ||
| current=`date "+%Y-%m-%d %H:%M:%S"` | ||
| timeStamp=`date -d "$current" +%s` | ||
| currentTimeStamp=$((timeStamp*1000+10#`date "+%N"`/1000000)) | ||
|
|
||
| # Temporally set to mlu370 | ||
| card_type="MLU370-S4" | ||
|
|
||
| # Default repo name | ||
| repo_name="triton-linalg" | ||
| # Repo ci root path | ||
| repo_root="/home/user1/${repo_name}_ci/" | ||
| if [ ! -d $repo_root ];then | ||
| mkdir $repo_root | ||
| fi | ||
| # Repo ci requests path | ||
| requests_path="$repo_root/requests" | ||
| if [ ! -d $requests_path ];then | ||
| mkdir $requests_path | ||
| fi | ||
|
|
||
| # Gen name of this ci | ||
| request_name="${repo_name}_${pr_id}_${currentTimeStamp}_${card_type}.rqt" | ||
|
|
||
| # Gen file and dir for this request | ||
| request_root="$repo_root/$request_name/" | ||
| sub_logs_path="$request_root/sub_logs/" | ||
|
|
||
|
|
||
| if [ ! -d $request_root ];then | ||
| mkdir $request_root | ||
| fi | ||
|
|
||
| if [ ! -d $sub_logs_path ];then | ||
| mkdir $sub_logs_path | ||
| fi | ||
|
|
||
| echo "working" > "$request_root/status" | ||
| chmod o+w "$request_root/status" | ||
|
|
||
| if [ ! -f "$request_root/log" ];then | ||
| touch "$request_root/log" | ||
| fi | ||
|
|
||
| chmod o+w "$request_root/log" | ||
|
|
||
| if [ ! -f "$request_root/log_list" ];then | ||
| touch "$request_root/log_list" | ||
| fi | ||
|
|
||
| chmod o+w "$request_root/log_list" | ||
|
|
||
| # Gen request file. | ||
|
|
||
| echo "repo:${repo_name}" > "$requests_path/${request_name}" | ||
| echo "pr_id:${pr_id}" >> "$requests_path/${request_name}" | ||
| echo "timestamp:${currentTimeStamp}" >> "$requests_path/${request_name}" | ||
|
|
||
| # change dir group for server and client, or when server/client try to delete request, ftp may raise error. | ||
| # start script | ||
| python3 .github/ci_script/file_guard.py "$request_root/status" "$request_root/log" & | ||
| python3 .github/ci_script/combine_log.py "$request_root/log" "$request_root/log_list" "$request_root/sub_logs" "$request_root/status" & | ||
|
|
||
| wait | ||
|
|
||
| status=$( head -n +1 ${request_root}/status ) | ||
|
|
||
| if [ "$status" != "success" ];then | ||
| echo "${status}" | ||
| exit -1 | ||
| else | ||
| echo "${status}" | ||
| exit 0 | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| name: triton-linalg_ci | ||
|
|
||
| on: | ||
| push: | ||
| branches: [master] | ||
| pull_request: | ||
| branches: [master] | ||
| jobs: | ||
| test: | ||
| strategy: | ||
| matrix: | ||
| triton-linalg_version : [v1.1.1] | ||
| runs-on: self-hosted | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| submodules: 'true' | ||
|
|
||
| - name: run_triton-linalg_ci | ||
| run: > | ||
| bash .github/ci_script/triton-linalg-ci_script.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.