diff --git a/constants.py b/constants.py index bda822d..2b03d14 100644 --- a/constants.py +++ b/constants.py @@ -101,3 +101,11 @@ ] VERS_LARGE_ENTITIES_SEPARATION_STARTED = 74 + +ACCESS_TOKEN = '' +REPO_NAME = 'shavar-prod-lists' +TARGET_BRANCH = 'updated_fingerprinting_list' +FINGERPRINTING_FILE_PATH = 'normalized-lists/base-fingerprinting-track.json' +COMMIT_MESSAGE = 'Updating fingerprinting list' +PR_TITLE = 'Updating fingerprinting list' +PR_DESCRIPTION = '' diff --git a/lists2safebrowsing.py b/lists2safebrowsing.py index 81c6bc3..17aa783 100755 --- a/lists2safebrowsing.py +++ b/lists2safebrowsing.py @@ -299,6 +299,11 @@ def write_safebrowsing_blocklist(domains, output_name, allow_list, log_file, hashdata_bytes += 32 publishing += 1 + if output_name == "base-fingerprinting-track-digest256": + domain_list = list(previous_domains) + with open("base-fingerprinting-track.json", 'w') as f: + json.dump(domain_list, f, indent=4) + # Write safebrowsing-list format header output_string = "a:%u:32:%s\n" % (chunk, hashdata_bytes) output_string += ''.join(output) diff --git a/publish2cloud.py b/publish2cloud.py index 865b124..651ba00 100644 --- a/publish2cloud.py +++ b/publish2cloud.py @@ -19,8 +19,16 @@ PRE_DNT_SECTIONS, LARGE_ENTITIES_SECTIONS, WHITELIST_SECTIONS, + ACCESS_TOKEN, + REPO_NAME, + TARGET_BRANCH, + FINGERPRINTING_FILE_PATH, + COMMIT_MESSAGE, + PR_DESCRIPTION, + PR_TITLE ) from packaging import version as p_version +from github import Github, UnknownObjectException CONFIG = ConfigParser.SafeConfigParser(os.environ) CONFIG.read(['shavar_list_creation.ini']) @@ -256,6 +264,33 @@ def publish_to_remote_settings(config, section): put_new_record_remote_settings(config, section, record_data) print('Uploaded to remote settings: %s' % list_name) +def update_fingerprinting_json(): + g = Github(ACCESS_TOKEN) + repo = g.get_user().get_repo(REPO_NAME) + list_of_branches = list(repo.get_branches()) + + for branch in list_of_branches: + branch_name = str(branch.name) + + #creating a new branch + branch_repo = repo.get_branch(branch_name) + branch_for_pr = 'refs/heads/' + TARGET_BRANCH + "_" + branch_name + branch_for_pr_name = TARGET_BRANCH + "_" + branch_name + repo.create_git_ref(ref=branch_for_pr, sha=branch_repo.commit.sha) + + #getting file + with open('base-fingerprinting-track.json', 'r') as file: + data = file.read() + + #creating a new commit + try: + contents = repo.get_contents(FINGERPRINTING_FILE_PATH, ref=branch_for_pr_name) + repo.update_file(contents.path, COMMIT_MESSAGE, data, contents.sha, branch=branch_for_pr_name) + except UnknownObjectException: + repo.create_file(FINGERPRINTING_FILE_PATH, COMMIT_MESSAGE, data, branch=branch_for_pr_name) + + #creating a pull request + pr = repo.create_pull(title=PR_TITLE, body=PR_DESCRIPTION, head=branch_for_pr_name, base=branch_name) def publish_to_cloud(config, chunknum, check_versioning=None): # Optionally upload to S3. If s3_upload is set, then s3_bucket and s3_key diff --git a/requirements.txt b/requirements.txt index 02a9871..bd9237f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ requests==2.10.0 trackingprotection_tools==0.4.6 packaging==19.2 setuptools==40.8.0 +PyGithub==1.45