- Sponsor
-
Notifications
You must be signed in to change notification settings - Fork 54
Move org deletion to background job with access to backend ops classes #2098
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
+404
−16
Merged
Changes from all commits
Commits
Show all changes
13 commits
Select commit
Hold shift + click to select a range
50bd569
WIP: Add delete org background job with access to org ops
tw4l 17c9cfc
Refine test
tw4l 7797fd7
Improve error handling
tw4l 8cb2b5e
WIP: Rename background-job template and load secrets
tw4l 33daefa
Add failed commented out attempt to use default namespace
tw4l c48d184
Add necessary secrets and volumes to crawler namespace
tw4l d6ecc62
Get background jobs working
tw4l 9c2c52f
Fix HTTP method for get job from all orgs
tw4l b038543
Move background jobs to default namespace
tw4l e96f082
Temporarily return replica jobs to crawlers namespace for storage sec…
tw4l df3c414
Merge branch 'main' into issue-1898-bg-jobs-org-methods
tw4l d601507
Add type annotation for namespace
tw4l be14e54
Update docstring
tw4l File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,144 @@ | ||
""" entrypoint module for background jobs """ | ||
|
||
import asyncio | ||
import os | ||
import sys | ||
import traceback | ||
from uuid import UUID | ||
|
||
from .crawlmanager import CrawlManager | ||
from .db import init_db | ||
from .emailsender import EmailSender | ||
|
||
# from .utils import register_exit_handler | ||
from .models import BgJobType | ||
|
||
from .basecrawls import BaseCrawlOps | ||
from .invites import InviteOps | ||
from .users import init_user_manager | ||
from .orgs import OrgOps | ||
from .colls import CollectionOps | ||
from .crawlconfigs import CrawlConfigOps | ||
from .crawls import CrawlOps | ||
from .profiles import ProfileOps | ||
from .storages import StorageOps | ||
from .webhooks import EventWebhookOps | ||
from .background_jobs import BackgroundJobOps | ||
from .pages import PageOps | ||
|
||
job_type = os.environ.get("BG_JOB_TYPE") | ||
oid = os.environ.get("OID") | ||
|
||
|
||
# ============================================================================ | ||
# pylint: disable=too-many-function-args, duplicate-code, too-many-locals | ||
async def main(): | ||
"""main init""" | ||
email = EmailSender() | ||
crawl_manager = None | ||
|
||
dbclient, mdb = init_db() | ||
|
||
invite_ops = InviteOps(mdb, email) | ||
|
||
user_manager = init_user_manager(mdb, email, invite_ops) | ||
|
||
org_ops = OrgOps(mdb, invite_ops, user_manager) | ||
|
||
event_webhook_ops = EventWebhookOps(mdb, org_ops) | ||
|
||
# pylint: disable=import-outside-toplevel | ||
if not os.environ.get("KUBERNETES_SERVICE_HOST"): | ||
print( | ||
"Sorry, the Browsertrix Backend must be run inside a Kubernetes environment.\ | ||
Kubernetes not detected (KUBERNETES_SERVICE_HOST is not set), Exiting" | ||
) | ||
sys.exit(1) | ||
|
||
crawl_manager = CrawlManager() | ||
|
||
storage_ops = StorageOps(org_ops, crawl_manager) | ||
|
||
background_job_ops = BackgroundJobOps( | ||
mdb, email, user_manager, org_ops, crawl_manager, storage_ops | ||
) | ||
|
||
profile_ops = ProfileOps( | ||
mdb, org_ops, crawl_manager, storage_ops, background_job_ops | ||
) | ||
|
||
crawl_config_ops = CrawlConfigOps( | ||
dbclient, | ||
mdb, | ||
user_manager, | ||
org_ops, | ||
crawl_manager, | ||
profile_ops, | ||
) | ||
|
||
coll_ops = CollectionOps(mdb, crawl_manager, org_ops, event_webhook_ops) | ||
|
||
base_crawl_ops = BaseCrawlOps( | ||
mdb, | ||
user_manager, | ||
org_ops, | ||
crawl_config_ops, | ||
coll_ops, | ||
storage_ops, | ||
event_webhook_ops, | ||
background_job_ops, | ||
) | ||
|
||
crawl_ops = CrawlOps( | ||
crawl_manager, | ||
mdb, | ||
user_manager, | ||
org_ops, | ||
crawl_config_ops, | ||
coll_ops, | ||
storage_ops, | ||
event_webhook_ops, | ||
background_job_ops, | ||
) | ||
|
||
page_ops = PageOps(mdb, crawl_ops, org_ops, storage_ops) | ||
|
||
base_crawl_ops.set_page_ops(page_ops) | ||
crawl_ops.set_page_ops(page_ops) | ||
|
||
background_job_ops.set_ops(crawl_ops, profile_ops) | ||
|
||
org_ops.set_ops(base_crawl_ops, profile_ops, coll_ops, background_job_ops) | ||
|
||
user_manager.set_ops(org_ops, crawl_config_ops, base_crawl_ops) | ||
|
||
background_job_ops.set_ops(base_crawl_ops, profile_ops) | ||
|
||
crawl_config_ops.set_coll_ops(coll_ops) | ||
|
||
# Run job | ||
if job_type == BgJobType.DELETE_ORG: | ||
if not oid: | ||
print("Org id missing, quitting") | ||
return 1 | ||
org = await org_ops.get_org_by_id(UUID(oid)) | ||
if not org: | ||
print("Org id invalid, quitting") | ||
return 1 | ||
|
||
try: | ||
await org_ops.delete_org_and_data(org, user_manager) | ||
return 0 | ||
# pylint: disable=broad-exception-caught | ||
except Exception: | ||
traceback.print_exc() | ||
return 1 | ||
|
||
print(f"Provided job type {job_type} not currently supported") | ||
return 1 | ||
|
||
|
||
# # ============================================================================ | ||
if __name__ == "__main__": | ||
return_code = asyncio.run(main()) | ||
sys.exit(return_code) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
apiVersion: batch/v1 | ||
kind: Job | ||
metadata: | ||
name: "{{ id }}" | ||
labels: | ||
role: "background-job" | ||
job_type: {{ job_type }} | ||
btrix.org: {{ oid }} | ||
|
||
spec: | ||
ttlSecondsAfterFinished: 0 | ||
backoffLimit: 3 | ||
template: | ||
spec: | ||
restartPolicy: Never | ||
priorityClassName: bg-job | ||
podFailurePolicy: | ||
rules: | ||
- action: FailJob | ||
onExitCodes: | ||
containerName: btrixbgjob | ||
operator: NotIn | ||
values: [0] | ||
|
||
volumes: | ||
- name: ops-configs | ||
secret: | ||
secretName: ops-configs | ||
|
||
containers: | ||
- name: btrixbgjob | ||
image: {{ backend_image }} | ||
imagePullPolicy: {{ pull_policy }} | ||
env: | ||
- name: BG_JOB_TYPE | ||
value: {{ job_type }} | ||
|
||
- name: OID | ||
value: {{ oid }} | ||
|
||
envFrom: | ||
- configMapRef: | ||
name: backend-env-config | ||
- secretRef: | ||
name: mongo-auth | ||
|
||
volumeMounts: | ||
- name: ops-configs | ||
mountPath: /ops-configs/ | ||
|
||
command: ["python3", "-m", "btrixcloud.main_bg"] | ||
|
||
resources: | ||
limits: | ||
memory: "200Mi" | ||
|
||
requests: | ||
memory: "200Mi" | ||
cpu: "50m" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.