Skip to content

Commit

Permalink
basic support for events
Browse files Browse the repository at this point in the history
  • Loading branch information
khimaros committed Jun 14, 2021
1 parent 70dffca commit f2987b1
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 2 deletions.
25 changes: 25 additions & 0 deletions github_to_sqlite/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,31 @@ def contributors(db_path, repos, auth):
utils.ensure_db_shape(db)


@cli.command()
@click.argument(
"db_path",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
required=True,
)
@click.argument("namespaces", type=str, nargs=-1)
@click.option(
"-a",
"--auth",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
default="auth.json",
help="Path to auth.json token file",
)
def events(db_path, namespaces, auth):
"Save events for the specified namespaces"
db = sqlite_utils.Database(db_path)
token = load_token(auth)
for ns in namespaces:
events = utils.fetch_events(ns, token)
utils.save_events(db, events, ns)
time.sleep(1)
utils.ensure_db_shape(db)


@cli.command()
@click.argument(
"db_path",
Expand Down
46 changes: 44 additions & 2 deletions github_to_sqlite/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,8 +305,14 @@ def save_repo(db, repo):
for key, value in repo.items()
if (key == "html_url") or not key.endswith("url")
}
to_save["owner"] = save_user(db, to_save["owner"])
to_save["license"] = save_license(db, to_save["license"])
if "owner" in to_save:
to_save["owner"] = save_user(db, to_save["owner"])
else:
to_save["owner"] = None
if "license" in to_save:
to_save["license"] = save_license(db, to_save["license"])
else:
to_save["license"] = None
if "organization" in to_save:
to_save["organization"] = save_user(db, to_save["organization"])
else:
Expand Down Expand Up @@ -396,6 +402,13 @@ def fetch_contributors(repo, token=None):
yield from contributors


def fetch_events(ns, token=None):
headers = make_headers(token)
url = "https://api.github.com/{}/events".format(ns)
for events in paginate(url, headers):
yield from events


def fetch_tags(repo, token=None):
headers = make_headers(token)
url = "https://api.github.com/repos/{}/tags".format(repo)
Expand Down Expand Up @@ -564,6 +577,35 @@ def save_contributors(db, contributors, repo_id):
)


def save_events(db, events, ns):
event_rows_to_add = []
for event in events:
user_id = save_user(db, event["actor"])
repo_id = save_repo(db, event["repo"])
created_at = event["created_at"]
public = bool(event["public"])
event_rows_to_add.append({
"user_id": user_id,
"repo_id": repo_id,
"created_at": created_at,
"public": public,
"type": event["type"],
"payload": event["payload"],
})
db["events"].insert_all(
event_rows_to_add,
pk=(
"repo_id",
"user_id",
),
foreign_keys=[
("repo_id", "repos", "id"),
("user_id", "users", "id")
],
replace=True,
)


def save_tags(db, tags, repo_id):
if not db["tags"].exists():
db["tags"].create(
Expand Down

0 comments on commit f2987b1

Please sign in to comment.