Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Generating fingerprinting list
Browse files Browse the repository at this point in the history
svensevenslow committed Apr 15, 2020

Verified

This commit was signed with the committer’s verified signature.
snyk-bot Snyk bot
1 parent bd0f384 commit e719ec8
Showing 4 changed files with 49 additions and 0 deletions.
8 changes: 8 additions & 0 deletions constants.py
Original file line number Diff line number Diff line change
@@ -101,3 +101,11 @@
]

VERS_LARGE_ENTITIES_SEPARATION_STARTED = 74

ACCESS_TOKEN = ''
REPO_NAME = 'shavar-prod-lists'
TARGET_BRANCH = 'updated_fingerprinting_list'
FINGERPRINTING_FILE_PATH = 'normalized-lists/base-fingerprinting-track.json'
COMMIT_MESSAGE = 'Updating fingerprinting list'
PR_TITLE = 'Updating fingerprinting list'
PR_DESCRIPTION = ''
5 changes: 5 additions & 0 deletions lists2safebrowsing.py
Original file line number Diff line number Diff line change
@@ -299,6 +299,11 @@ def write_safebrowsing_blocklist(domains, output_name, allow_list, log_file,
hashdata_bytes += 32
publishing += 1

if output_name == "base-fingerprinting-track-digest256":
domain_list = list(previous_domains)
with open("base-fingerprinting-track.json", 'w') as f:
json.dump(domain_list, f, indent=4)

# Write safebrowsing-list format header
output_string = "a:%u:32:%s\n" % (chunk, hashdata_bytes)
output_string += ''.join(output)
35 changes: 35 additions & 0 deletions publish2cloud.py
Original file line number Diff line number Diff line change
@@ -19,8 +19,16 @@
PRE_DNT_SECTIONS,
LARGE_ENTITIES_SECTIONS,
WHITELIST_SECTIONS,
ACCESS_TOKEN,
REPO_NAME,
TARGET_BRANCH,
FINGERPRINTING_FILE_PATH,
COMMIT_MESSAGE,
PR_DESCRIPTION,
PR_TITLE
)
from packaging import version as p_version
from github import Github, UnknownObjectException

CONFIG = ConfigParser.SafeConfigParser(os.environ)
CONFIG.read(['shavar_list_creation.ini'])
@@ -256,6 +264,33 @@ def publish_to_remote_settings(config, section):
put_new_record_remote_settings(config, section, record_data)
print('Uploaded to remote settings: %s' % list_name)

def update_fingerprinting_json():
g = Github(ACCESS_TOKEN)
repo = g.get_user().get_repo(REPO_NAME)
list_of_branches = list(repo.get_branches())

for branch in list_of_branches:
branch_name = str(branch.name)

#creating a new branch
branch_repo = repo.get_branch(branch_name)
branch_for_pr = 'refs/heads/' + TARGET_BRANCH + "_" + branch_name
branch_for_pr_name = TARGET_BRANCH + "_" + branch_name
repo.create_git_ref(ref=branch_for_pr, sha=branch_repo.commit.sha)

#getting file
with open('base-fingerprinting-track.json', 'r') as file:
data = file.read()

#creating a new commit
try:
contents = repo.get_contents(FINGERPRINTING_FILE_PATH, ref=branch_for_pr_name)
repo.update_file(contents.path, COMMIT_MESSAGE, data, contents.sha, branch=branch_for_pr_name)
except UnknownObjectException:
repo.create_file(FINGERPRINTING_FILE_PATH, COMMIT_MESSAGE, data, branch=branch_for_pr_name)

#creating a pull request
pr = repo.create_pull(title=PR_TITLE, body=PR_DESCRIPTION, head=branch_for_pr_name, base=branch_name)

def publish_to_cloud(config, chunknum, check_versioning=None):
# Optionally upload to S3. If s3_upload is set, then s3_bucket and s3_key
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -4,3 +4,4 @@ requests==2.10.0
trackingprotection_tools==0.4.6
packaging==19.2
setuptools==40.8.0
PyGithub==1.45

0 comments on commit e719ec8

Please sign in to comment.