feat(core): common method to get website info

packages/core/src/const.ts

@@ -8,7 +8,7 @@ export const GITHUB_RAW_URL = 'https://raw.githubusercontent.com'
 export const NOTION_API_URL = 'https://api.notion.com/v1'
 export const OPENAI_CHAT_API = 'https://api.openai.com'
 export const MAX_TOKEN_LENGTH = 2048
-export const SUMMARIZE_PROMPTS = `Please summarize content and then classify it to 1-5 types of classification. Classification names should be short and no explanation or description is needed.
+export const SUMMARIZE_PROMPTS = `You are a summarize professer. Please summarize content in detail and then classify it to 1-5 types of classification. Classification names should be short and no explanation or description is needed.
 You only speak JSON. Do not write text that isn't JSON. The JSON keys must be English word of "summary" and "categories".
 Classification names used with array data.
 The JSON format must be:
@@ -17,11 +17,11 @@ The JSON format must be:
   "categories": ["XXX","YYY","ZZZ"]
 }
 `
-export const USER_PROMPTS = `Please answer in {language}. The Content is {webprompts}:
+export const USER_PROMPTS = `Please summarize in {language}. The Content is {webprompts}:
 =====
 {content}
 =====
-Please answer in {language}.`
+Please summarize in {language}. Remember You only speak JSON.`
 
 export const ANSWER_IN_LANGUAGE: { [key in PromptsLanguage]: string } = {
   'de': 'German',

packages/core/src/types.ts

@@ -15,6 +15,9 @@ export interface WebsiteMeta {
   ogImage?: string
   prompts?: string
 }
+export interface CommonMeta extends WebsiteMeta {
+  favicon?: string
+}
 // TODO: remove web meta type to webHub
 export interface GithubRepoMeta extends WebsiteMeta {
   username?: string

packages/core/src/webInfo/index.ts

@@ -1,9 +1,6 @@
 import { $fetch } from 'ofetch'
 import type { Routes, WebInfoData, WebLoaderUrls } from '../types'
 import { getDomain } from '../utils'
-import type { OGInfo } from './ogInfo'
-
-export * from './ogInfo'
 
 export class WebInfoByApi {
   constructor(fields: { stargramHub: string; headers?: Record<string, string> }) {
@@ -36,15 +33,13 @@ export class WebInfoByApi {
 }
 
 export class WebInfo {
-  constructor(fields: { routes: Routes; headers?: Record<string, string>; ogInfo?: OGInfo }) {
+  constructor(fields: { routes: Routes; headers?: Record<string, string> }) {
     this.headers = fields.headers
     this.routes = fields.routes
-    this.ogInfo = fields.ogInfo
   }
 
   private routes: Routes
   private headers?: Record<string, string>
-  private ogInfo
 
   async call(urls: WebLoaderUrls) {
     const domain = getDomain(urls.webUrl)
@@ -66,8 +61,18 @@ export class WebInfo {
       }
       throw new Error(`${router.name} error: Not supported website.`)
     }
-    else if (this.ogInfo) {
-      return await this.ogInfo.call(urls.webUrl)
+    else if (this.routes.common) {
+      const router = this.routes.common
+      if (router.paths) {
+        for (let i = 0; i < router.paths.length; i++) {
+          const pathInfo = router.paths[i]
+          const loaderUrls = pathInfo.filter(urls)
+          if (loaderUrls) {
+            const info = await pathInfo.loader(loaderUrls, this.headers)
+            return info
+          }
+        }
+      }
     }
 
     throw new Error('Stargram error: Not supported website.')

packages/webHub/package.json

@@ -39,14 +39,28 @@
     "test": "vitest"
   },
   "peerDependencies": {
-    "@stargram/core": "workspace:*"
+    "@mozilla/readability": "^0.4.4",
+    "@stargram/core": "workspace:*",
+    "htmlparser2": "^9.0.0",
+    "jsdom": "^22.1.0"
   },
   "peerDependenciesMeta": {
     "@stargram/core": {
       "optional": false
+    },
+    "@mozilla/readability": {
+      "optional": false
+    },
+    "htmlparser2": {
+      "optional": false
+    },
+    "jsdom": {
+      "optional": false
     }
   },
   "devDependencies": {
-    "@stargram/generate-routes": "workspace:*"
+    "@stargram/generate-routes": "workspace:*",
+    "@types/htmlparser2": "^3.10.3",
+    "@types/jsdom": "^21.1.1"
   }
 }

packages/webHub/src/routes-auto-generate.ts

@@ -1,10 +1,15 @@
 // Auto Generated by '@stargram/generate-routes'
 import type { Routes } from '@stargram/core'
+import { router as common } from './website/common/router'
+import { pathInfo as commonAny } from './website/common/paths/any'
 import { router as githubcom } from './website/github.com/router'
 import { pathInfo as githubcomRepo } from './website/github.com/paths/repo'
 import { router as twittercom } from './website/twitter.com/router'
 import { pathInfo as twittercomTweet } from './website/twitter.com/paths/tweet'
 
+common.paths = [
+  commonAny,
+].sort((a, b) => (b.sequence ?? 0) - (a.sequence ?? 0))
 githubcom.paths = [
   githubcomRepo,
 ].sort((a, b) => (b.sequence ?? 0) - (a.sequence ?? 0))
@@ -13,6 +18,7 @@ twittercom.paths = [
 ].sort((a, b) => (b.sequence ?? 0) - (a.sequence ?? 0))
 
 export const routes: Routes = {
+  'common': common,
   'github.com': githubcom,
   'twitter.com': twittercom,
 }