66import logging
77import psycopg2
88import dateutil
9+ import argparse
910
1011def init_logger ():
1112 LOGLEVEL = os .environ .get ('LOGLEVEL' , 'INFO' ).upper ()
1213 logging .basicConfig (level = LOGLEVEL ,
1314 format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' ,
1415 datefmt = '%m-%d-%Y %H:%M:%S' )
1516
17+ def make_parser ():
18+ parser = argparse .ArgumentParser ()
19+ parser .add_argument ('-r' , '--repository-name' , type = str , required = True ,
20+ help = 'Repository name in OWNER/REPOSITORY format' )
21+ parser .add_argument ('--run-id' , type = str , required = True ,
22+ help = 'Workflow Run ID' )
23+
24+ return parser
25+
1626def create_db_tables (conn , cur ):
17- cur .execute ('''CREATE TABLE IF NOT EXISTS github_workflow_runs_test (
18- id SERIAL,
19- run_id BIGINT PRIMARY KEY ,
27+ cur .execute ('''CREATE TABLE IF NOT EXISTS workflow_runs (
28+ id SERIAL PRIMARY KEY ,
29+ run_id BIGINT,
2030 html_url TEXT,
2131 name VARCHAR(255),
2232 run_started_at TIMESTAMP,
33+ created_at TIMESTAMP,
34+ updated_at TIMESTAMP,
2335 triggering_actor_login VARCHAR(255),
2436 conclusion VARCHAR(25),
25- run_number INT,
2637 event VARCHAR(50),
2738 run_attempt INT,
2839 repository_full_name VARCHAR(255),
2940 head_repository_full_name VARCHAR(255),
3041 head_branch VARCHAR(255),
3142 status VARCHAR(25),
3243 display_title TEXT,
33- path TEXT
44+ path TEXT,
45+ total_duration_seconds INT
3446 );
3547 ''' )
36- cur .execute ('''CREATE TABLE IF NOT EXISTS github_workflow_jobs_test (
37- id SERIAL,
38- job_id BIGINT PRIMARY KEY ,
39- parent_run_id BIGINT REFERENCES github_workflow_runs_test(run_id) ,
48+ cur .execute ('''CREATE TABLE IF NOT EXISTS workflow_jobs (
49+ id SERIAL PRIMARY KEY ,
50+ job_id BIGINT,
51+ parent_run_id BIGINT,
4052 html_url TEXT,
4153 name VARCHAR(255),
4254 created_at TIMESTAMP,
@@ -47,12 +59,14 @@ def create_db_tables(conn, cur):
4759 runner_name VARCHAR(255),
4860 status VARCHAR(25),
4961 conclusion VARCHAR(25),
50- head_branch VARCHAR(255)
62+ head_branch VARCHAR(255),
63+ run_attempt INT,
64+ workflow_name TEXT
5165 );
5266 ''' )
53- cur .execute ('''CREATE TABLE IF NOT EXISTS github_workflow_steps_test (
67+ cur .execute ('''CREATE TABLE IF NOT EXISTS workflow_steps (
5468 id SERIAL PRIMARY KEY,
55- parent_job_id BIGINT REFERENCES github_workflow_jobs_test(job_id) ,
69+ parent_job_id BIGINT,
5670 name VARCHAR(255),
5771 conclusion VARCHAR(25),
5872 number INT,
@@ -65,20 +79,16 @@ def create_db_tables(conn, cur):
6579
6680def main ():
6781 init_logger ()
68-
82+ parser = make_parser ()
83+ args = parser .parse_args ()
6984 logger = logging .getLogger (__name__ )
7085
7186 github_token = os .environ .get ('GITHUB_TOKEN' )
7287 if not github_token :
7388 raise ValueError ('GITHUB_TOKEN environment variable is not set!' )
7489
75- run_id = os .environ .get ('RUN_ID' )
76- if not run_id :
77- raise ValueError ('RUN_ID environment variable is not set!' )
78-
79- repo_name = os .environ .get ('GITHUB_REPOSITORY' )
80- if not repo_name :
81- raise ValueError ('GITHUB_REPOSITORY environment variable is not set!' )
90+ run_id = args .run_id
91+ repo_name = args .repository_name
8292
8393
8494 # this should be specified in runner's env
@@ -102,18 +112,31 @@ def main():
102112 repo = g .get_repo (repo_name )
103113
104114 run = repo .get_workflow_run (int (run_id ))
105-
106- workflow_data_query = f'''INSERT INTO github_workflow_runs_test(
115+ logger .debug ('Processing run ID %s - %s' , run_id , run .name )
116+ if run .status != 'completed' :
117+ logger .error ('Run %s is not completed! Only completed runs should be in the database' , run_id )
118+ raise SystemExit (1 )
119+
120+ # We rely on the following assumptions:
121+ # - The workflow run is completed. When run.status != 'completed' we should not add it to the database
122+ # theoretically the second attempt can be triggerred right after the completion of the first one
123+ # or while the runner which executes this script is deploying
124+ #
125+ # - Job's queued duration equals "job.started_at - job.created_at" if started_at > created_at.
126+ # Otherwise the job should not be added to the database
127+ total_duration_seconds = round (run .timing ().run_duration_ms / 1000 )
128+ workflow_data_query = f'''INSERT INTO workflow_runs(
107129 run_id, html_url, name,
108- run_started_at, triggering_actor_login, conclusion,
109- run_number, event, run_attempt, repository_full_name,
110- head_branch, display_title, path)
130+ run_started_at, created_at, updated_at, triggering_actor_login, conclusion,
131+ event, run_attempt, repository_full_name,
132+ head_branch, display_title, path, total_duration_seconds )
111133 VALUES(
112134 '{ run_id } { run .html_url } { run .name } { run .run_started_at }
135+ '{ run .created_at } { run .updated_at }
113136 '{ run .raw_data ['triggering_actor' ]['login' ]}
114- '{ run .conclusion } { run .run_number } ', ' { run . event }
137+ '{ run .conclusion } { run .event }
115138 '{ run .run_attempt } { run .raw_data ['repository' ]['full_name' ]}
116- '{ run .head_branch } { run .display_title } { run .path }
139+ '{ run .head_branch } { run .display_title } { run .path } , ' { total_duration_seconds } '
117140 );
118141 '''
119142
@@ -122,10 +145,15 @@ def main():
122145
123146 for job in run .jobs ():
124147 job_id = job .id
148+ logger .debug ('Processing job %s' , job .name )
125149 queued_duration_seconds = 0
126150 duration_seconds = 0
127151
128152 job_created_at_date = dateutil .parser .parse (job .raw_data ['created_at' ])
153+ if job_created_at_date > job .started_at :
154+ logger .warning ('Skipping job %s of run %s - most likely a stub \
155+ job .name , run_id )
156+ continue
129157
130158 queued_duration_timedelta = job .started_at - job_created_at_date
131159 queued_duration_seconds = round (queued_duration_timedelta .total_seconds ())
@@ -134,27 +162,30 @@ def main():
134162 duration_seconds = round (duration_timedelta .total_seconds ())
135163
136164 job_data_query = f'''
137- INSERT INTO github_workflow_jobs_test (
165+ INSERT INTO workflow_jobs (
138166 job_id, parent_run_id, html_url, name,
139167 created_at, started_at, completed_at,
140168 queued_duration_seconds, duration_seconds,
141- runner_name, status, conclusion, head_branch)
169+ runner_name, status, conclusion, head_branch,
170+ run_attempt, workflow_name
171+ )
142172 VALUES(
143173 '{ job_id } { run_id } { job .html_url } { job .name }
144174 '{ job .raw_data ['created_at' ]} { job .started_at } { job .completed_at }
145175 '{ queued_duration_seconds } { duration_seconds }
146176 '{ job .raw_data ['runner_name' ]} { job .status } { job .conclusion }
147- '{ job .raw_data ['head_branch' ]}
177+ '{ job .raw_data ['head_branch' ]} , ' { job . raw_data [ 'run_attempt' ] } ', ' { job . raw_data [ 'workflow_name' ] } '
148178 );
149179 '''
150180 logger .debug ('Job query: %s' , job_data_query )
151181 cur .execute (job_data_query )
152182 for step in job .steps :
183+ logger .debug ('Processing step %s' , step .name )
153184 duration_seconds_timedelta = step .completed_at - step .started_at
154185 duration_seconds = round (duration_seconds_timedelta .total_seconds ())
155186
156187 step_data_query = f'''
157- INSERT INTO github_workflow_steps_test (
188+ INSERT INTO workflow_steps (
158189 parent_job_id, name, conclusion,
159190 number, started_at, completed_at,
160191 duration_seconds)
0 commit comments