From 87277469e040bcaf57ca2345070fe5a28182bb56 Mon Sep 17 00:00:00 2001 From: chadcarlson Date: Fri, 28 Jun 2024 09:16:29 -0400 Subject: [PATCH 1/9] Small readme change. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 1ad85ab..564ca7e 100644 --- a/README.md +++ b/README.md @@ -313,3 +313,7 @@ There are far more concepts than could be explored in a single demo project, tal But we have some of the best minds in web development and computing ready and excited to help with your side project, experiment, or next big idea. [Join us on Discord and less us help you get going](https://discord.gg/PkMc2pVCDV)! + +### 6. Testing notifications + +TBD \ No newline at end of file From 8c54849ac66973089ad141859b0a8935c32ec45e Mon Sep 17 00:00:00 2001 From: chadcarlson Date: Fri, 28 Jun 2024 15:45:48 -0400 Subject: [PATCH 2/9] Add mount and basic watchdog. --- .upsun/config.yaml | 144 +++++++++++++++++++---------------- apps/nodejs/.gitignore | 5 +- apps/nodejs/requirements.txt | 8 ++ apps/nodejs/watch.py | 67 ++++++++++++++++ 4 files changed, 158 insertions(+), 66 deletions(-) create mode 100644 apps/nodejs/requirements.txt create mode 100644 apps/nodejs/watch.py diff --git a/.upsun/config.yaml b/.upsun/config.yaml index 1fd7322..465634d 100644 --- a/.upsun/config.yaml +++ b/.upsun/config.yaml @@ -1,37 +1,37 @@ applications: - main_app: - source: - root: apps/main - stack: - - nodejs@22 - - nodePackages.npm - hooks: - build: | - set -e - npm install - npm run setup-env - npm run build - web: - locations: - /: - root: "dist" - passthru: true - index: - - index.html + # main_app: + # source: + # root: apps/main + # stack: + # - nodejs@22 + # - nodePackages.npm + # hooks: + # build: | + # set -e + # npm install + # npm run setup-env + # npm run build + # web: + # locations: + # /: + # root: "dist" + # passthru: true + # index: + # - index.html - bun_app: - source: - root: apps/bun - stack: - - bun@1 - hooks: - build: | - set -e - bun install - web: - commands: - start: bun start + # bun_app: + # source: + # root: apps/bun + # stack: + # - bun@1 + # hooks: + # build: | + # set -e + # bun install + # web: + # commands: + # start: bun start nodejs_app: source: @@ -39,9 +39,12 @@ applications: stack: - nodejs@22 - nodePackages.npm + - python@3.12 + - python312Packages.pip hooks: build: | set -e + pip install -r requirements.txt npm install npm run build @@ -52,22 +55,33 @@ applications: commands: start: npm run start - deno_app: - source: - root: apps/deno - stack: - - deno - variables: - env: - DENO_DIR: "cache" - hooks: - build: | - set -e - deno cache package.json - deno compile --allow-env --allow-net --allow-read main.ts - web: + mounts: + 'test-mount': + source: storage + + workers: + watchdog: commands: - start: deno task start + start: | + python watch.py $PLATFORM_APP_DIR/test-mount + + + # deno_app: + # source: + # root: apps/deno + # stack: + # - deno + # variables: + # env: + # DENO_DIR: "cache" + # hooks: + # build: | + # set -e + # deno cache package.json + # deno compile --allow-env --allow-net --allow-read main.ts + # web: + # commands: + # start: deno task start services: database: @@ -78,28 +92,28 @@ routes: "https://{default}/": type: upstream primary: true - upstream: "main_app:http" + upstream: "nodejs_app:http" "https://www.{default}": type: redirect to: "https://{default}/" - "https://{default}/bun": - type: upstream - upstream: "bun_app:http" - "https://www.{default}/bun": - type: redirect - to: "https://{default}/bun" + # "https://{default}/bun": + # type: upstream + # upstream: "bun_app:http" + # "https://www.{default}/bun": + # type: redirect + # to: "https://{default}/bun" - "https://{default}/nodejs": - type: upstream - upstream: "nodejs_app:http" - "https://www.{default}/nodejs": - type: redirect - to: "https://{default}/nodejs" + # "https://{default}/nodejs": + # type: upstream + # upstream: "nodejs_app:http" + # "https://www.{default}/nodejs": + # type: redirect + # to: "https://{default}/nodejs" - "https://{default}/deno": - type: upstream - upstream: "deno_app:http" - "https://www.{default}/deno": - type: redirect - to: "https://{default}/deno" + # "https://{default}/deno": + # type: upstream + # upstream: "deno_app:http" + # "https://www.{default}/deno": + # type: redirect + # to: "https://{default}/deno" diff --git a/apps/nodejs/.gitignore b/apps/nodejs/.gitignore index 53c37a1..42c93c1 100644 --- a/apps/nodejs/.gitignore +++ b/apps/nodejs/.gitignore @@ -1 +1,4 @@ -dist \ No newline at end of file +dist +env +test_mount +__pycache__ diff --git a/apps/nodejs/requirements.txt b/apps/nodejs/requirements.txt new file mode 100644 index 0000000..2338585 --- /dev/null +++ b/apps/nodejs/requirements.txt @@ -0,0 +1,8 @@ +certifi==2024.6.2 +charset-normalizer==3.3.2 +idna==3.7 +nose==1.3.7 +PyYAML==6.0.1 +requests==2.32.3 +urllib3==2.2.2 +watchdog==4.0.1 diff --git a/apps/nodejs/watch.py b/apps/nodejs/watch.py new file mode 100644 index 0000000..2169316 --- /dev/null +++ b/apps/nodejs/watch.py @@ -0,0 +1,67 @@ +# https://github.com/gorakhargosh/watchdog +# https://pypi.org/project/requests/ +# https://philipkiely.com/code/python_watchdog.html +# + +import os +import sys +import time +import datetime +import requests +import logging +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +class Watcher: + + def __init__(self, directory=".", handler=FileSystemEventHandler()): + self.observer = Observer() + self.handler = handler + self.directory = directory + + def run(self): + self.observer.schedule( + self.handler, self.directory, recursive=True) + self.observer.start() + print("\nWatcher Running in {}/\n".format(self.directory)) + try: + while True: + time.sleep(1) + except: + self.observer.stop() + self.observer.join() + print("\nWatcher Terminated\n") + +class MyHandler(FileSystemEventHandler): + + webhook_url = "https://webhook.site/cc04606b-25eb-45d5-8e55-31fa4438db0a" + + if os.environ.get("WATCH_WEBHOOK_URL") is not None: + webhook_url = os.environ["WATCH_WEBHOOK_URL"] + + logging.basicConfig(level=logging.INFO, + format='%(asctime)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + + def on_any_event(self, event): + if not event.is_directory: + event_data = { + "src_path": event.src_path, + "dest_path": event.dest_path, + "event_type": event.event_type, + "is_directory": event.is_directory, + "is_synthetic": event.is_synthetic, + "timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S%z") + } + logging.info(f'{event.event_type.upper()!r}: {event_data!r}') + resp = requests.post(self.webhook_url, json = event_data) + alert_data = { + "status_code": resp.status_code, + "url": resp.url + } + logging.info(f'{"FORWARDED".upper()!r}: {alert_data!r}') + +if __name__=="__main__": + watchpath = sys.argv[1] if len(sys.argv) > 1 else '.' + w = Watcher(watchpath, MyHandler()) + w.run() From 5fed56692c526b8a67c4e28d86c098061209b7e3 Mon Sep 17 00:00:00 2001 From: chadcarlson Date: Fri, 28 Jun 2024 15:48:18 -0400 Subject: [PATCH 3/9] indent typo. --- .upsun/config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.upsun/config.yaml b/.upsun/config.yaml index 465634d..1bfc9d8 100644 --- a/.upsun/config.yaml +++ b/.upsun/config.yaml @@ -59,11 +59,11 @@ applications: 'test-mount': source: storage - workers: - watchdog: - commands: - start: | - python watch.py $PLATFORM_APP_DIR/test-mount + workers: + watchdog: + commands: + start: | + python watch.py $PLATFORM_APP_DIR/test-mount # deno_app: From c1d2af5a5c397ffb897584c4a49bb688811470a0 Mon Sep 17 00:00:00 2001 From: chadcarlson Date: Mon, 1 Jul 2024 11:47:36 -0400 Subject: [PATCH 4/9] Filter out less interesting events. --- apps/nodejs/watch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/nodejs/watch.py b/apps/nodejs/watch.py index 2169316..19e59f0 100644 --- a/apps/nodejs/watch.py +++ b/apps/nodejs/watch.py @@ -44,7 +44,8 @@ class MyHandler(FileSystemEventHandler): datefmt='%Y-%m-%d %H:%M:%S') def on_any_event(self, event): - if not event.is_directory: + uninteresting_events = ["closed", "opened"] + if and ( not event.is_directory ) (event.event_type not in uninteresting_events): event_data = { "src_path": event.src_path, "dest_path": event.dest_path, From 4d9dc1a706a9f6d1f0098a550042030a9f71d924 Mon Sep 17 00:00:00 2001 From: chadcarlson Date: Mon, 1 Jul 2024 12:24:06 -0400 Subject: [PATCH 5/9] syntax error. --- apps/nodejs/watch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/nodejs/watch.py b/apps/nodejs/watch.py index 19e59f0..6b440bc 100644 --- a/apps/nodejs/watch.py +++ b/apps/nodejs/watch.py @@ -1,7 +1,7 @@ # https://github.com/gorakhargosh/watchdog # https://pypi.org/project/requests/ # https://philipkiely.com/code/python_watchdog.html -# +# inotify only available on Linux, which doesn't match my local env import os import sys @@ -45,7 +45,7 @@ class MyHandler(FileSystemEventHandler): def on_any_event(self, event): uninteresting_events = ["closed", "opened"] - if and ( not event.is_directory ) (event.event_type not in uninteresting_events): + if ( not event.is_directory ) and ( event.event_type not in uninteresting_events ): event_data = { "src_path": event.src_path, "dest_path": event.dest_path, From 60f582741df7fd0f63100d446ff56de63b322c5a Mon Sep 17 00:00:00 2001 From: chadcarlson Date: Mon, 1 Jul 2024 12:51:53 -0400 Subject: [PATCH 6/9] Add an explicit network storage mount. --- .upsun/config.yaml | 83 +++++++--------------------------------------- 1 file changed, 12 insertions(+), 71 deletions(-) diff --git a/.upsun/config.yaml b/.upsun/config.yaml index 1bfc9d8..aa5e1a3 100644 --- a/.upsun/config.yaml +++ b/.upsun/config.yaml @@ -1,38 +1,5 @@ applications: - # main_app: - # source: - # root: apps/main - # stack: - # - nodejs@22 - # - nodePackages.npm - # hooks: - # build: | - # set -e - # npm install - # npm run setup-env - # npm run build - # web: - # locations: - # /: - # root: "dist" - # passthru: true - # index: - # - index.html - - # bun_app: - # source: - # root: apps/bun - # stack: - # - bun@1 - # hooks: - # build: | - # set -e - # bun install - # web: - # commands: - # start: bun start - nodejs_app: source: root: apps/nodejs @@ -58,35 +25,30 @@ applications: mounts: 'test-mount': source: storage + source_path: files + 'test-mountb': + source: service + service: network-storage + source_path: filesb workers: watchdog: commands: start: | python watch.py $PLATFORM_APP_DIR/test-mount + watchdogb: + commands: + start: | + python watch.py $PLATFORM_APP_DIR/test-mountb - # deno_app: - # source: - # root: apps/deno - # stack: - # - deno - # variables: - # env: - # DENO_DIR: "cache" - # hooks: - # build: | - # set -e - # deno cache package.json - # deno compile --allow-env --allow-net --allow-read main.ts - # web: - # commands: - # start: deno task start - services: database: type: mariadb:10.4 + ns: + type: network-storage:2.0 + routes: "https://{default}/": @@ -96,24 +58,3 @@ routes: "https://www.{default}": type: redirect to: "https://{default}/" - - # "https://{default}/bun": - # type: upstream - # upstream: "bun_app:http" - # "https://www.{default}/bun": - # type: redirect - # to: "https://{default}/bun" - - # "https://{default}/nodejs": - # type: upstream - # upstream: "nodejs_app:http" - # "https://www.{default}/nodejs": - # type: redirect - # to: "https://{default}/nodejs" - - # "https://{default}/deno": - # type: upstream - # upstream: "deno_app:http" - # "https://www.{default}/deno": - # type: redirect - # to: "https://{default}/deno" From 401b1107120a3de78fd50a6ec1d64af9e57ae04a Mon Sep 17 00:00:00 2001 From: chadcarlson Date: Mon, 1 Jul 2024 12:57:28 -0400 Subject: [PATCH 7/9] wrong service name. --- .upsun/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.upsun/config.yaml b/.upsun/config.yaml index aa5e1a3..b2b431d 100644 --- a/.upsun/config.yaml +++ b/.upsun/config.yaml @@ -28,7 +28,7 @@ applications: source_path: files 'test-mountb': source: service - service: network-storage + service: ns source_path: filesb workers: From bf65949d0b6875bdb109bcd7e5d312e7490ecbbf Mon Sep 17 00:00:00 2001 From: chadcarlson Date: Mon, 1 Jul 2024 14:13:04 -0400 Subject: [PATCH 8/9] Use polling observer instead. --- .upsun/config.yaml | 2 -- apps/nodejs/watch.py | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.upsun/config.yaml b/.upsun/config.yaml index b2b431d..29da3b4 100644 --- a/.upsun/config.yaml +++ b/.upsun/config.yaml @@ -40,8 +40,6 @@ applications: commands: start: | python watch.py $PLATFORM_APP_DIR/test-mountb - - services: database: type: mariadb:10.4 diff --git a/apps/nodejs/watch.py b/apps/nodejs/watch.py index 6b440bc..f673bc3 100644 --- a/apps/nodejs/watch.py +++ b/apps/nodejs/watch.py @@ -9,13 +9,15 @@ import datetime import requests import logging -from watchdog.observers import Observer +# from watchdog.observers import Observer +from watchdog.observers.polling import PollingObserver from watchdog.events import FileSystemEventHandler class Watcher: def __init__(self, directory=".", handler=FileSystemEventHandler()): - self.observer = Observer() + # self.observer = Observer() + self.observer = PollingObserver() self.handler = handler self.directory = directory From 5a36aaf315613f3ddc161b8d08d998adff37eb03 Mon Sep 17 00:00:00 2001 From: chadcarlson Date: Mon, 1 Jul 2024 14:55:44 -0400 Subject: [PATCH 9/9] Cleanup + add inotify. --- .upsun/config.yaml | 27 +++++++-------------------- apps/nodejs/watch.py | 4 ++++ 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/.upsun/config.yaml b/.upsun/config.yaml index 29da3b4..8182de4 100644 --- a/.upsun/config.yaml +++ b/.upsun/config.yaml @@ -8,45 +8,32 @@ applications: - nodePackages.npm - python@3.12 - python312Packages.pip + - inotify-tools hooks: build: | set -e pip install -r requirements.txt npm install npm run build - - relationships: - database: - web: commands: start: npm run start - + # access: + # ssh: admin mounts: - 'test-mount': + 'sensitive-data': source: storage - source_path: files - 'test-mountb': - source: service - service: ns - source_path: filesb - workers: watchdog: commands: start: | - python watch.py $PLATFORM_APP_DIR/test-mount - watchdogb: - commands: - start: | - python watch.py $PLATFORM_APP_DIR/test-mountb + python watch.py $PLATFORM_APP_DIR/sensitive-data + relationships: + database: services: database: type: mariadb:10.4 - ns: - type: network-storage:2.0 - routes: "https://{default}/": diff --git a/apps/nodejs/watch.py b/apps/nodejs/watch.py index f673bc3..15723bb 100644 --- a/apps/nodejs/watch.py +++ b/apps/nodejs/watch.py @@ -1,6 +1,9 @@ # https://github.com/gorakhargosh/watchdog # https://pypi.org/project/requests/ # https://philipkiely.com/code/python_watchdog.html +# Combined the above script with the one on the repo +# _But_ we're dealing with Network mounted drives https://www.pythonanywhere.com/forums/topic/7521/ +# ... so we need to use the PollingObserver class https://python-watchdog.readthedocs.io/en/stable/_modules/watchdog/observers/polling.html # inotify only available on Linux, which doesn't match my local env import os @@ -48,6 +51,7 @@ class MyHandler(FileSystemEventHandler): def on_any_event(self, event): uninteresting_events = ["closed", "opened"] if ( not event.is_directory ) and ( event.event_type not in uninteresting_events ): + print(dir(event)) event_data = { "src_path": event.src_path, "dest_path": event.dest_path,