Dev deploy, reject bots (#1)

* dev

* dev deploy

* more staging

* CNAME

* cname

* index.staging.html

* dev-deploy

* index.staging.html

* add whiletrue to dev sites

* dev deploy

* dev deploy

* dev deploy

* dev deploy

* reject bot

.github/workflows/develop.yml

@@ -0,0 +1,24 @@
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Build projects
+
+on:
+  push:
+    branches: [ dev ]
+  workflow_dispatch:
+jobs:
+  build-server:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: '16'
+    - name: Build
+      run: |
+        git config --global user.email "adam.kariv@gmail.com"
+        git config --global user.name "Adam Kariv"
+        npm install -g npm@8
+        npm install
+        ./dev-deploy.sh

CNAME

@@ -0,0 +1 @@
+budgetkey.whiletrue.industries

angular.json

@@ -68,6 +68,13 @@
               "extractLicenses": false,
               "sourceMap": true,
               "namedChunks": true
+            },
+            "staging": {
+              "index": {
+                "input": "projects/budgetkey/src/index.staging.html",
+                "output": "index.html"
+              },
+              "outputHashing": "all"
             }
           },
           "defaultConfiguration": "production"

dev-deploy.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+git checkout dev && \
+(git branch -D dist || true) && \
+git checkout -b dist && \
+rm .gitignore && \
+cd utils && ./mk_bubbles.sh && cd .. && \
+npm run build-dev && \
+cp dist/budgetkey/browser/index.html dist/budgetkey/browser/404.html && \
+cp CNAME dist/budgetkey/browser/ && \
+git add dist/budgetkey/browser/ && \
+git commit -m dist && \
+(git branch -D gh-pages || true) && \
+git subtree split --prefix dist/budgetkey/browser -b gh-pages && \
+git push -f origin gh-pages:gh-pages && \
+git checkout dev && \
+git branch -D gh-pages && \
+git branch -D dist && \
+git checkout . 

package.json

@@ -5,6 +5,7 @@
     "ng": "ng",
     "start": "ng serve",
     "build": "ng build",
+    "build-dev": "ng build --configuration staging",
     "watch": "ng build --watch --configuration development",
     "test": "ng test",
     "dev:ssr": "ng run budgetkey:serve-ssr",

projects/budgetkey/server.ts

@@ -31,13 +31,19 @@ export function app(): express.Express {
 
   // All regular routes use the Universal engine
   server.get('*', (req, res) => {
+    const user_agent = req.headers['user-agent'];
+    if (user_agent === 'thesis-research-bot') {
+      res.send('יש לנו קובץ עם כל המידע, בבקשה לא לעשות לנו סקרייפינג!');
+      console.log(`${new Date().toISOString()} | ${res.statusCode} | REJECT | ${req.url} | ${user_agent}`);
+      return;
+    }
     const hostname = req.headers['x-forwarded-host'] || req.hostname;
     if (req.path === '/' && hostname.indexOf('socialpro.org.il') > -1) {
       res.redirect('https://www.socialpro.org.il/i/units/gov_social_service_unit/main?theme=soproc');
       return;
     }
     res.render(indexHtml, { req, providers: [{ provide: APP_BASE_HREF, useValue: req.baseUrl }] }, (err, html) => {
-      console.log(`${new Date().toISOString()} | ${res.statusCode} | ${err?.name || 'OK'} | ${req.url} | ${req.headers['user-agent']}}`);      
+      console.log(`${new Date().toISOString()} | ${res.statusCode} | ${err?.name || 'OK'} | ${req.url} | ${user_agent}`);
       if (err) {
         if (res.statusCode !== 302) {
           console.log('ERR:', err);

projects/budgetkey/src/app/common-components/components/searchbar/bk-search-bar.component.ts

@@ -99,7 +99,7 @@ export class BkSearchBar implements OnChanges, AfterViewInit, OnInit {
     }
     urlParams = params.toString();
     const url = '/s/?' + urlParams;
-    if (hostname !== 'next.obudget.org' && hostname.indexOf('localhost') !== 0) {
+    if (hostname !== 'next.obudget.org' && hostname.indexOf('localhost') !== 0 && hostname.indexOf('whiletrue') < 0) {
       return 'https://next.obudget.org' + url;
     } else {
       return url;

projects/budgetkey/src/index.staging.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html lang="he">
+<head>
+  <meta charset="utf-8">
+  <title>מפתח התקציב</title>
+  <base href="/"/>
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"/>
+  <meta name="referrer" content="origin" />
+  <meta name="image" content="https://next.obudget.org/assets/common/img/obudget_key@2x.png"/>
+
+  <link rel="icon" type="image/x-icon" href="favicon.ico"/>  
+</head>
+<body>
+  <app-root></app-root>
+</body>
+</html>

utils/mk_bubbles.js

@@ -305,9 +305,11 @@ function fetch_all() {
     });
 }
 
+console.log('RUNNING');
 fetch_all()
   .then((data) => {
     const content = `export const bubbles: any = ` + JSON.stringify(data, undefined, 4) + `;`;
     const filename = '../projects/budgetkey/src/app/main-page/bubbles.ts'
+    console.log('DONE');
     fs.writeFileSync(filename, content);
-  });
+  });