chore(env): add back env

.aws/deploy/backend-task-definition.prod.json

@@ -103,6 +103,7 @@
           "valueFrom": "PROD_INCOMING_QUEUE_URL"
         },
         { "name": "JWT_SECRET", "valueFrom": "PROD_JWT_SECRET" },
+        { "name": "KEYCDN_API_KEY", "valueFrom": "PROD_KEYCDN_API_KEY" },
         {
           "name": "MAX_NUM_OTP_ATTEMPTS",
           "valueFrom": "PROD_MAX_NUM_OTP_ATTEMPTS"
@@ -132,6 +133,11 @@
           "name": "REDIRECT_URI",
           "valueFrom": "PROD_REDIRECT_URI"
         },
+        {
+          "name": "REDIRECTION_REPO_GITHUB_TOKEN",
+          "valueFrom": "PROD_REDIRECTION_REPO_GITHUB_TOKEN"
+        },
+        { "name": "REDIS_HOST", "valueFrom": "PROD_REDIS_HOST" },
         {
           "name": "SESSION_SECRET",
           "valueFrom": "PROD_SESSION_SECRET"

.aws/deploy/backend-task-definition.staging.json

@@ -112,6 +112,7 @@
           "valueFrom": "STAGING_INCOMING_QUEUE_URL"
         },
         { "name": "JWT_SECRET", "valueFrom": "STAGING_JWT_SECRET" },
+        { "name": "KEYCDN_API_KEY", "valueFrom": "STAGING_KEYCDN_API_KEY" },
         {
           "name": "MAX_NUM_OTP_ATTEMPTS",
           "valueFrom": "STAGING_MAX_NUM_OTP_ATTEMPTS"
@@ -141,6 +142,11 @@
           "name": "REDIRECT_URI",
           "valueFrom": "STAGING_REDIRECT_URI"
         },
+        {
+          "name": "REDIRECTION_REPO_GITHUB_TOKEN",
+          "valueFrom": "STAGING_REDIRECTION_REPO_GITHUB_TOKEN"
+        },
+        { "name": "REDIS_HOST", "valueFrom": "STAGING_REDIS_HOST" },
         {
           "name": "SESSION_SECRET",
           "valueFrom": "STAGING_SESSION_SECRET"

.aws/deploy/support-task-definition.prod.json

@@ -128,6 +128,10 @@
           "name": "REDIRECT_URI",
           "valueFrom": "PROD_REDIRECT_URI"
         },
+        {
+          "name": "REDIRECTION_REPO_GITHUB_TOKEN",
+          "valueFrom": "PROD_REDIRECTION_REPO_GITHUB_TOKEN"
+        },
         { "name": "REDIS_HOST", "valueFrom": "PROD_REDIS_HOST" },
         {
           "name": "SESSION_SECRET",

.aws/deploy/support-task-definition.staging.json

@@ -137,6 +137,10 @@
           "name": "REDIRECT_URI",
           "valueFrom": "STAGING_REDIRECT_URI"
         },
+        {
+          "name": "REDIRECTION_REPO_GITHUB_TOKEN",
+          "valueFrom": "STAGING_REDIRECTION_REPO_GITHUB_TOKEN"
+        },
         { "name": "REDIS_HOST", "valueFrom": "STAGING_REDIS_HOST" },
         {
           "name": "SESSION_SECRET",

src/config/config.ts

@@ -254,6 +254,13 @@ const config = convict({
       format: "required-string",
       default: "",
     },
+    redirectionRepoGithubToken: {
+      doc: "Github access to read opengovsg/isomer-redirection",
+      env: "REDIRECTION_REPO_GITHUB_TOKEN",
+      sensitive: true,
+      format: "required-string",
+      default: "",
+    },
   },
   dataDog: {
     env: {

src/constants/constants.ts

@@ -78,6 +78,7 @@ export const REDIRECTION_SERVER_IPS = [
 ]
 export const ALLOWED_DNS_ERROR_CODES = ["ENOTFOUND", "ENODATA"]
 
+export const BUILT_WITH_ISOMER_LOGO = `<p class="footer-credits"><a href="https://www.isomer.gov.sg/" target="_blank" rel="noreferrer">Created with <img src="/assets/img/isomer_logo.svg" alt="Isomer Logo"></a>`
 export const DNS_INDIRECTION_DOMAIN = "hostedon.isomer.gov.sg"
 export const DNS_KEYCDN_SUFFIX = "kxcdn.com"
 export const DNS_CNAME_SUFFIXES = ["cloudfront.net", DNS_KEYCDN_SUFFIX]

src/monitoring/MonitoringWorker.ts

@@ -12,7 +12,7 @@ import MonitoringError from "@root/errors/MonitoringError"
 import { gb } from "@root/middleware/featureFlag"
 import LaunchesService from "@root/services/identity/LaunchesService"
 import { dnsMonitor } from "@root/utils/dns-utils"
-import { isMonitoringEnabled } from "@root/utils/growthbook-utils"
+import { getMonitoringConfig } from "@root/utils/growthbook-utils"
 import promisifyPapaParse from "@root/utils/papa-parse"
 
 const IsomerHostedDomainType = {
@@ -106,14 +106,16 @@ export default class MonitoringWorker {
    * @returns List of redirection domains that are listed in the isomer-redirection repository
    */
   getRedirectionDomains() {
-    const SYSTEM_GITHUB_TOKEN = config.get("github.systemToken")
+    const REDIRECTION_REPO_GITHUB_TOKEN = config.get(
+      "github.redirectionRepoGithubToken"
+    )
     // seems to be a bug in typing, this is a direct
     // copy paste from the octokit documentation
     // https://octokit.github.io/rest.js/v20#automatic-retries
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const OctokitRetry = Octokit.plugin(retry as any)
     const octokitWithRetry: Octokit = new OctokitRetry({
-      auth: SYSTEM_GITHUB_TOKEN,
+      auth: REDIRECTION_REPO_GITHUB_TOKEN,
       request: { retries: 5 },
     })
 
@@ -175,7 +177,9 @@ export default class MonitoringWorker {
   }
 
   driver() {
-    if (!isMonitoringEnabled(gb)) return okAsync("Monitoring Service disabled")
+    const monitoringConfig = getMonitoringConfig(gb)
+    if (!monitoringConfig.isEnabled)
+      return okAsync("Monitoring Service disabled")
     const start = Date.now()
     this.monitoringWorkerLogger.info("Monitoring service started")
 
@@ -196,6 +200,15 @@ export default class MonitoringWorker {
             date: new Date(),
           },
         })
+        reportCardErr.map((err) =>
+          this.monitoringWorkerLogger.error({
+            message: "Error running monitoring service",
+            meta: {
+              dnsCheckerResult: err,
+              date: new Date(),
+            },
+          })
+        )
       })
       .orElse(() => okAsync([]))
       .andThen(() => {

src/server.ts

@@ -81,7 +81,6 @@ import { mailer } from "@services/utilServices/MailClient"
 
 import { apiLogger } from "./middleware/apiLogger"
 import { NotificationOnEditHandler } from "./middleware/notificationOnEditHandler"
-import MonitoringService from "./monitoring"
 import getAuthenticatedSubrouter from "./routes/v2/authenticated"
 import { ReviewsRouter } from "./routes/v2/authenticated/review"
 import getAuthenticatedSitesSubrouter from "./routes/v2/authenticatedSites"

src/types/featureFlags.ts

@@ -7,13 +7,18 @@ export interface CloudmersiveConfigType {
   timeout: number
 }
 
+export interface MonitoringConfig {
+  isEnabled: boolean
+  whitelistedRepos: string[]
+}
+
 export interface FeatureFlags {
   is_build_times_reduction_enabled: boolean
   is_ggs_enabled: boolean
   is_show_staging_build_status_enabled: boolean
   is_cloudmersive_enabled: CloudmersiveConfigType
   is_local_diff_enabled: boolean
-  is_monitoring_enabled: boolean
+  is_monitoring_enabled: MonitoringConfig
 }
 
 // List of attributes we set in GrowthBook Instance in auth middleware

src/utils/dns-utils.ts

@@ -1,14 +1,19 @@
 import dns from "node:dns/promises"
 
-import { err, ok, okAsync, Result, ResultAsync } from "neverthrow"
+import { err, errAsync, ok, okAsync, Result, ResultAsync } from "neverthrow"
 
 import {
+  BUILT_WITH_ISOMER_LOGO,
   DNS_CNAME_SUFFIXES,
   DNS_INDIRECTION_DOMAIN,
   DNS_KEYCDN_SUFFIX,
   REDIRECTION_SERVER_IPS,
 } from "@root/constants"
+import MonitoringError from "@root/errors/MonitoringError"
 import logger from "@root/logger/logger"
+import { gb } from "@root/middleware/featureFlag"
+
+import { getMonitoringConfig } from "./growthbook-utils"
 
 export function checkCname(domain: string) {
   return ResultAsync.fromPromise(dns.resolveCname(domain), () => {
@@ -23,6 +28,11 @@ export function checkCname(domain: string) {
         return okAsync(null)
       }
 
+      //! todo: remove, this is some colima weird logic
+      if (cname.length === 1 && cname[0] === domain) {
+        return okAsync(null)
+      }
+
       return okAsync(cname[0])
     })
     .orElse(() => okAsync(null))
@@ -118,7 +128,7 @@ export default function getDnsCheckerMessage(
       )
     }
     if (isIntermediateValid && isRedirectionValid) {
-      return err(
+      return ok(
         `The domain ${domain} is *all valid*! Although it is directly pointing to our CDN hosting provider and not the indirection layer. The apex domain \`${redirectionDomain}\` is also correctly configured to our redirection service.`
       )
     }
@@ -167,23 +177,36 @@ export default function getDnsCheckerMessage(
         : "no A records"
     }`
   }
+
   return err(responseMeta)
 }
 
 export function dnsMonitor(domain: string): ResultAsync<string, string> {
+  if (!domain) {
+    return okAsync("Empty domain provided")
+  }
   return checkCname(domain)
     .andThen((cname) => {
+      if (cname) {
+        return ResultAsync.combine([okAsync(domain), okAsync(cname)])
+      }
       // Original domain does not have a CNAME record, check if the www
       // version has a valid CNAME record
       if (!cname && !domain.startsWith("www.")) {
         const cnameDomain = `www.${domain}`
-        return ResultAsync.combine([
-          okAsync(cnameDomain),
-          checkCname(cnameDomain),
-        ])
+        return checkCname(cnameDomain).andThen((cnameDomainRes) => {
+          if (cnameDomainRes) {
+            return ResultAsync.combine([
+              okAsync(cnameDomain),
+              okAsync(cnameDomainRes),
+            ])
+          }
+          // this might be a redirection only domain
+          return ResultAsync.combine([okAsync(domain), okAsync(null)])
+        })
       }
-
-      return ResultAsync.combine([okAsync(domain), okAsync(cname)])
+      // might be a redirection only domain
+      return ResultAsync.combine([okAsync(domain), okAsync(null)])
     })
     .andThen(([cnameDomain, cnameRecord]) => {
       // Original and www version of the domain do not have a CNAME record,
@@ -245,8 +268,8 @@ export function dnsMonitor(domain: string): ResultAsync<string, string> {
           redirectionDomain,
         },
         redirection,
-      ]) =>
-        getDnsCheckerMessage(
+      ]) => {
+        const dnsCheckerMessage = getDnsCheckerMessage(
           domain,
           cnameDomain,
           redirectionDomain,
@@ -255,5 +278,50 @@ export function dnsMonitor(domain: string): ResultAsync<string, string> {
           indirectionRecords,
           redirection
         )
+
+        const domainCheckerRes = dnsCheckerMessage.isOk()
+          ? dnsCheckerMessage.value
+          : dnsCheckerMessage.error
+
+        return ResultAsync.fromPromise(
+          // using fetch as it has a convenient redirect option
+
+          fetch(`https://${domain}`),
+          () => new MonitoringError(`Failed to fetch site: ${domain}`)
+        )
+          .andThen((res) =>
+            ResultAsync.fromPromise(
+              res.text(),
+              () => new MonitoringError(`Failed to get text from response`)
+            )
+          )
+          .andThen((res) => {
+            const isStillHostingIsomerSite = res.includes(
+              BUILT_WITH_ISOMER_LOGO
+            )
+
+            if (isStillHostingIsomerSite) {
+              return okAsync(
+                `${domainCheckerRes}\nThe site is still being hosted with Isomer logo, and is likely still viewable by MOP`
+              )
+            }
+
+            const monitoringConfig = getMonitoringConfig(gb)
+
+            const isWeirdDomain = monitoringConfig.whitelistedRepos.includes(
+              domain
+            )
+            if (isWeirdDomain) {
+              return okAsync(
+                `${domainCheckerRes}\nThe site is not being hosted with Isomer logo, and is known to be an edge case.`
+              )
+            }
+            return errAsync(false)
+          })
+          .mapErr(
+            () =>
+              `${domainCheckerRes}\nThe site is **NOT** being hosted with Isomer logo, and is likely **NOT** viewable by MOP`
+          )
+      }
     )
 }

src/utils/growthbook-utils.ts

@@ -2,7 +2,11 @@ import { GrowthBook, setPolyfills } from "@growthbook/growthbook"
 
 import { config } from "@root/config/config"
 import { FEATURE_FLAGS } from "@root/constants/featureFlags"
-import { FeatureFlags, CloudmersiveConfigType } from "@root/types/featureFlags"
+import {
+  FeatureFlags,
+  CloudmersiveConfigType,
+  MonitoringConfig,
+} from "@root/types/featureFlags"
 
 const GROWTHBOOK_API_HOST = "https://cdn.growthbook.io"
 
@@ -68,9 +72,16 @@ export const isCloudmersiveEnabled = (
   )
 }
 
-export const isMonitoringEnabled = (
+export const getMonitoringConfig = (
   growthbook: GrowthBook<FeatureFlags> | undefined
-): boolean => {
-  if (!growthbook) return true
-  return growthbook.getFeatureValue(FEATURE_FLAGS.IS_MONITORING_ENABLED, true)
+): MonitoringConfig => {
+  const defaultConfig = {
+    isEnabled: false,
+    whitelistedRepos: [],
+  }
+  if (!growthbook) return defaultConfig
+  return growthbook.getFeatureValue(
+    FEATURE_FLAGS.IS_MONITORING_ENABLED,
+    defaultConfig
+  )
 }

support/index.ts

@@ -7,7 +7,7 @@ import { useSharedMiddleware } from "@common/middleware"
 import { config } from "@root/config/config"
 import logger from "@root/logger/logger"
 import MonitoringService from "@root/monitoring/MonitoringService"
-import MonitoringWorker from "@root/monitoring/monitoringWorker"
+import MonitoringWorker from "@root/monitoring/MonitoringWorker"
 
 import { ROUTE_VERSION } from "./constants"
 import { v2Router } from "./routes"
@@ -24,6 +24,9 @@ infraService.pollMessages()
 export const monitoringWorker = new MonitoringWorker({
   launchesService,
 })
+// dnsMonitor("isomer.gov.sg").mapErr(console.log).map(console.log)
+// todo: remove after testing
+monitoringWorker.driver()
 
 export const monitoringService = new MonitoringService({
   monitoringWorker,