diff --git a/firestore.indexes.json b/firestore.indexes.json index e54196f..d571900 100644 --- a/firestore.indexes.json +++ b/firestore.indexes.json @@ -1,7 +1,7 @@ { "indexes": [ { - "collectionGroup": "repoVersions", + "collectionGroup": "editorVersions", "queryScope": "COLLECTION", "fields": [ { @@ -19,7 +19,7 @@ ] }, { - "collectionGroup": "unityVersions", + "collectionGroup": "repoVersions", "queryScope": "COLLECTION", "fields": [ { diff --git a/functions/package-lock.json b/functions/package-lock.json index 96b83ef..aec65c5 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -218,6 +218,14 @@ "optional": true, "requires": { "semver": "^6.2.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "optional": true + } } }, "@grpc/proto-loader": { @@ -585,6 +593,12 @@ "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.3.tgz", "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==" }, + "@types/semver": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.4.tgz", + "integrity": "sha512-+nVsLKlcUCeMzD2ufHEYuJ9a2ovstb6Dp52A5VsoKxDXgvE051XgHI/33I1EymwkRGQkwnA0LkhnUzituGs4EQ==", + "dev": true + }, "@types/serve-static": { "version": "1.13.5", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.5.tgz", @@ -2219,6 +2233,14 @@ "retry-request": "^4.0.0", "semver": "^6.0.0", "walkdir": "^0.4.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "optional": true + } } }, "google-p12-pem": { @@ -2901,6 +2923,14 @@ "optional": true, "requires": { "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "optional": true + } } }, "media-typer": { @@ -3592,10 +3622,9 @@ } }, "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "optional": true + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==" }, "send": { "version": "0.17.1", diff --git a/functions/package.json b/functions/package.json index c1c9b14..dc917c7 100644 --- a/functions/package.json +++ b/functions/package.json @@ -24,12 +24,14 @@ "httpie": "^1.1.2", "jsdom": "^16.4.0", "lodash": "^4.17.20", - "node-fetch": "^2.6.1" + "node-fetch": "^2.6.1", + "semver": "^7.3.2" }, "devDependencies": { "@types/jsdom": "^16.2.4", "@types/node": "^14.11.5", "@types/node-fetch": "^2.5.7", + "@types/semver": "^7.3.4", "@typescript-eslint/eslint-plugin": "^3.9.1", "@typescript-eslint/parser": "^3.8.0", "eslint": "^7.6.0", diff --git a/functions/src/api/repoVerions.ts b/functions/src/api/repoVerions.ts index eb4ad55..e35efe9 100644 --- a/functions/src/api/repoVerions.ts +++ b/functions/src/api/repoVerions.ts @@ -1,7 +1,7 @@ import { Request } from 'firebase-functions/lib/providers/https'; import { Response } from 'express-serve-static-core'; import { firebase, functions } from '../config/firebase'; -import { RepoVersionInfo } from '../model/repoVersions'; +import { RepoVersionInfo } from '../model/repoVersionInfo'; export const repoVersions = functions.https.onRequest( async (request: Request, response: Response) => { diff --git a/functions/src/api/reportNewBuild.ts b/functions/src/api/reportNewBuild.ts index b10cf5a..1efe87c 100644 --- a/functions/src/api/reportNewBuild.ts +++ b/functions/src/api/reportNewBuild.ts @@ -6,7 +6,7 @@ import { BuildInfo, CiBuilds, ImageType } from '../model/ciBuilds'; import { CiJobs } from '../model/ciJobs'; import { Discord } from '../config/discord'; import { EditorVersionInfo } from '../model/editorVersionInfo'; -import { RepoVersionInfo } from '../model/repoVersions'; +import { RepoVersionInfo } from '../model/repoVersionInfo'; export const reportNewBuild = functions.https.onRequest(async (req: Request, res: Response) => { try { diff --git a/functions/src/cron/index.ts b/functions/src/cron/index.ts index 9a1c3f0..72891e4 100644 --- a/functions/src/cron/index.ts +++ b/functions/src/cron/index.ts @@ -2,6 +2,7 @@ import { EventContext } from 'firebase-functions'; import { firebase, functions } from '../config/firebase'; import { Discord } from '../config/discord'; import { ingestUnityVersions } from '../logic/ingestUnityVersions'; +import { ingestRepoVersions } from '../logic/ingestRepoVersions'; /** * CPU-time for pubSub is not part of the free quota, so we'll keep it light weight. @@ -24,5 +25,6 @@ export const trigger = functions.pubsub }); const routineTasks = async () => { + await ingestRepoVersions(); await ingestUnityVersions(); }; diff --git a/functions/src/logic/ingestRepoVersions/index.ts b/functions/src/logic/ingestRepoVersions/index.ts new file mode 100644 index 0000000..cd290b6 --- /dev/null +++ b/functions/src/logic/ingestRepoVersions/index.ts @@ -0,0 +1,21 @@ +import { firebase } from '../../config/firebase'; +import { Discord } from '../../config/discord'; +import { scrapeVersions } from './scrapeVersions'; +import { updateDatabase } from './updateDatabase'; + +export const ingestRepoVersions = async () => { + try { + const scrapedInfoList = await scrapeVersions(); + firebase.logger.info('Found versions', scrapedInfoList); + + await updateDatabase(scrapedInfoList); + } catch (err) { + const message = ` + Something went wrong while importing repository versions for unity-ci/docker: + ${err.message} (${err.status})\n${err.stackTrace} + `; + + firebase.logger.error(message); + await Discord.sendAlert(message); + } +}; diff --git a/functions/src/logic/ingestRepoVersions/scrapeVersions.ts b/functions/src/logic/ingestRepoVersions/scrapeVersions.ts new file mode 100644 index 0000000..3484a1b --- /dev/null +++ b/functions/src/logic/ingestRepoVersions/scrapeVersions.ts @@ -0,0 +1,47 @@ +import semver from 'semver'; +import { RepoVersionInfo } from '../../model/repoVersionInfo'; +import { GitHub } from '../../config/github'; + +export const scrapeVersions = async (): Promise => { + const gitHub = await GitHub.init(); + + const releases = await gitHub.repos.listReleases({ + owner: 'unity-ci', + repo: 'docker', + }); + + const versions = releases.data.map((release) => { + const { + id, + url, + name, + body: description, + tag_name: tagName, + author: { login: author }, + target_commitish: commitIsh, + } = release; + + const version = semver.valid(semver.coerce(tagName)); + if (!version) { + throw new Error("Assumed versions to always be parsable, but they're not."); + } + const major = semver.major(version); + const minor = semver.major(version); + const patch = semver.major(version); + + return { + version, + major, + minor, + patch, + id, + name, + description, + author, + commitIsh, + url, + }; + }); + + return versions; +}; diff --git a/functions/src/logic/ingestRepoVersions/updateDatabase.ts b/functions/src/logic/ingestRepoVersions/updateDatabase.ts new file mode 100644 index 0000000..e5f33e2 --- /dev/null +++ b/functions/src/logic/ingestRepoVersions/updateDatabase.ts @@ -0,0 +1,54 @@ +import { isMatch } from 'lodash'; +import { RepoVersionInfo } from '../../model/repoVersionInfo'; +import { firebase } from '../../config/firebase'; +import { Discord } from '../../config/discord'; + +const plural = (amount: number) => { + return amount === 1 ? 'version' : 'versions'; +}; + +export const updateDatabase = async (ingestedInfoList: RepoVersionInfo[]): Promise => { + const existingInfoList = await RepoVersionInfo.getAll(); + + const newVersions: RepoVersionInfo[] = []; + const updatedVersions: RepoVersionInfo[] = []; + + ingestedInfoList.forEach((ingestedInfo: RepoVersionInfo) => { + const { version } = ingestedInfo; + const existingVersion = existingInfoList.find((info) => info.version === version); + + if (!existingVersion) { + newVersions.push(ingestedInfo); + return; + } + + if (!isMatch(existingVersion, ingestedInfo)) { + updatedVersions.push(ingestedInfo); + return; + } + }); + + let message = ''; + + if (newVersions.length >= 1) { + await RepoVersionInfo.createMany(newVersions); + message += ` + ${newVersions.length} new repository ${plural(newVersions.length)} detected. + (${newVersions.map((version) => version.version).join(', ')}`; + } + + if (updatedVersions.length >= 1) { + await RepoVersionInfo.updateMany(updatedVersions); + message += ` + ${updatedVersions.length} updated repository ${plural(updatedVersions.length)} detected. + (${updatedVersions.map((version) => version.version).join(', ')})`; + } + + message = message.trimEnd(); + if (message.length >= 1) { + firebase.logger.info(message); + await Discord.sendMessageToMaintainers(message); + } else { + firebase.logger.info('Database is up-to-date. (no updated repo versions found)'); + } +}; diff --git a/functions/src/logic/ingestUnityVersions/updateDatabase.ts b/functions/src/logic/ingestUnityVersions/updateDatabase.ts index f28871f..194d183 100644 --- a/functions/src/logic/ingestUnityVersions/updateDatabase.ts +++ b/functions/src/logic/ingestUnityVersions/updateDatabase.ts @@ -1,7 +1,7 @@ -import { EditorVersionInfo } from '../../model/editorVersionInfo'; import { isMatch } from 'lodash'; import { firebase } from '../../config/firebase'; import { Discord } from '../../config/discord'; +import { EditorVersionInfo } from '../../model/editorVersionInfo'; const plural = (amount: number) => { return amount === 1 ? 'version' : 'versions'; @@ -32,12 +32,16 @@ export const updateDatabase = async (ingestedInfoList: EditorVersionInfo[]): Pro if (newVersions.length >= 1) { await EditorVersionInfo.createMany(newVersions); - message += `${newVersions.length} new ${plural(newVersions.length)} detected. `; + message += ` + ${newVersions.length} new Unity editor ${plural(newVersions.length)} detected. + (${newVersions.map((version) => version.version).join(', ')}`; } if (updatedVersions.length >= 1) { await EditorVersionInfo.updateMany(updatedVersions); - message += `${updatedVersions.length} updated ${plural(updatedVersions.length)} detected. `; + message += ` + ${updatedVersions.length} updated Unity editor ${plural(updatedVersions.length)} detected. + (${updatedVersions.map((version) => version.version).join(', ')})`; } message = message.trimEnd(); diff --git a/functions/src/model/ciJobs.ts b/functions/src/model/ciJobs.ts index a0382a3..0b57810 100644 --- a/functions/src/model/ciJobs.ts +++ b/functions/src/model/ciJobs.ts @@ -2,7 +2,7 @@ import { db, admin } from '../config/firebase'; import { EditorVersionInfo } from './editorVersionInfo'; import FieldValue = admin.firestore.FieldValue; import Timestamp = admin.firestore.Timestamp; -import { RepoVersionInfo } from './repoVersions'; +import { RepoVersionInfo } from './repoVersionInfo'; import { ImageType } from './ciBuilds'; const COLLECTION = 'ciJobs'; diff --git a/functions/src/model/ciVersionInfo.ts b/functions/src/model/ciVersionInfo.ts index 7b15a1c..d994800 100644 --- a/functions/src/model/ciVersionInfo.ts +++ b/functions/src/model/ciVersionInfo.ts @@ -1,7 +1,7 @@ import { db, admin, firebase } from '../config/firebase'; import Timestamp = admin.firestore.Timestamp; import { EditorVersionInfo } from './editorVersionInfo'; -import { RepoVersionInfo } from './repoVersions'; +import { RepoVersionInfo } from './repoVersionInfo'; const COLLECTION = 'builtVersions'; diff --git a/functions/src/model/editorVersionInfo.ts b/functions/src/model/editorVersionInfo.ts index c9544d2..c7e9114 100644 --- a/functions/src/model/editorVersionInfo.ts +++ b/functions/src/model/editorVersionInfo.ts @@ -1,7 +1,7 @@ import { db, admin, firebase } from '../config/firebase'; import Timestamp = admin.firestore.Timestamp; -const COLLECTION = 'unityVersions'; +const COLLECTION = 'editorVersions'; export interface EditorVersionInfo { version: string; @@ -42,11 +42,11 @@ export class EditorVersionInfo { return snapshot.docs.map((doc) => doc.data()) as EditorVersionInfo[]; }; - static createMany = async (versionInfoList: EditorVersionInfo[]) => { + static createMany = async (editorVersionList: EditorVersionInfo[]) => { try { const batch = db.batch(); - versionInfoList.forEach((versionInfo) => { + editorVersionList.forEach((versionInfo) => { const { version } = versionInfo; const ref = db.collection(COLLECTION).doc(version); @@ -56,7 +56,7 @@ export class EditorVersionInfo { await batch.commit(); } catch (err) { - firebase.logger.error('Error occurred during batch commit of new version', err); + firebase.logger.error('Error occurred during batch commit of new editor versions', err); } }; @@ -74,7 +74,7 @@ export class EditorVersionInfo { await batch.commit(); } catch (err) { - firebase.logger.error('Error occurred during batch commit of new version', err); + firebase.logger.error('Error occurred during batch commit of new editor versions', err); } }; } diff --git a/functions/src/model/repoVersions.ts b/functions/src/model/repoVersionInfo.ts similarity index 55% rename from functions/src/model/repoVersions.ts rename to functions/src/model/repoVersionInfo.ts index 2ac25b7..9ce334b 100644 --- a/functions/src/model/repoVersions.ts +++ b/functions/src/model/repoVersionInfo.ts @@ -1,13 +1,19 @@ -import { db, admin } from '../config/firebase'; +import { db, admin, firebase } from '../config/firebase'; import Timestamp = admin.firestore.Timestamp; const COLLECTION = 'repoVersions'; export interface RepoVersionInfo { + id: number; version: string; major: number; minor: number; - patch: string; + patch: number; + name: string; + description: string; + author: string; + url: string; + commitIsh: string; addedDate?: Timestamp; modifiedDate?: Timestamp; } @@ -22,6 +28,10 @@ export class RepoVersionInfo { .limit(1) .get(); + if (snapshot.docs.length <= 1) { + throw new Error('No repository versions have been ingested yet'); + } + return snapshot.docs[0].data() as RepoVersionInfo; }; @@ -75,4 +85,40 @@ export class RepoVersionInfo { { merge: true }, ); }; + + static createMany = async (repoVersionList: RepoVersionInfo[]) => { + try { + const batch = db.batch(); + + repoVersionList.forEach((versionInfo) => { + const { version } = versionInfo; + + const ref = db.collection(COLLECTION).doc(version); + const data = { ...versionInfo, addedDate: Timestamp.now(), modifiedDate: Timestamp.now() }; + batch.set(ref, data, { merge: false }); + }); + + await batch.commit(); + } catch (err) { + firebase.logger.error('Error occurred during batch commit of new repo versions', err); + } + }; + + static updateMany = async (repoVersionList: RepoVersionInfo[]) => { + try { + const batch = db.batch(); + + repoVersionList.forEach((versionInfo) => { + const { version } = versionInfo; + + const ref = db.collection(COLLECTION).doc(version); + const data = { ...versionInfo, modifiedDate: Timestamp.now() }; + batch.set(ref, data, { merge: true }); + }); + + await batch.commit(); + } catch (err) { + firebase.logger.error('Error occurred during batch commit of new repo versions', err); + } + }; }