Feature: Pick which branch to load the github data from (#1491)

app/app.component.ts

@@ -10,9 +10,9 @@ import { PermissionService } from "@batch-flask/ui/permission";
 import { registerIcons } from "app/config";
 import {
     AccountService, AuthorizationHttpService, AutoscaleFormulaService,
-    BatchLabsService, CommandService, NavigatorService, NcjTemplateService,
-    NodeService, PredefinedFormulaService, PricingService, PythonRpcService, SSHKeyService,
-    SettingsService, SubscriptionService, ThemeService, VmSizeService,
+    BatchLabsService, CommandService, GithubDataService, NavigatorService,
+    NcjTemplateService, NodeService, PredefinedFormulaService, PricingService, PythonRpcService,
+    SSHKeyService, SettingsService, SubscriptionService, ThemeService, VmSizeService,
 } from "app/services";
 
 @Component({
@@ -32,6 +32,7 @@ export class AppComponent implements OnInit {
         private accountService: AccountService,
         private navigatorService: NavigatorService,
         private subscriptionService: SubscriptionService,
+        private githubDataService: GithubDataService,
         private nodeService: NodeService,
         private sshKeyService: SSHKeyService,
         batchLabsService: BatchLabsService,
@@ -49,6 +50,7 @@ export class AppComponent implements OnInit {
     ) {
         this.autoscaleFormulaService.init();
         this.settingsService.init();
+        this.githubDataService.init();
         this.sshKeyService.init();
         this.commandService.init();
         this.pricingService.init();

app/components/market/home/market.component.ts

@@ -5,7 +5,7 @@ import { Subscription } from "rxjs";
 
 import { autobind } from "@batch-flask/core";
 import { Application } from "app/models";
-import { NcjTemplateService } from "app/services";
+import { GithubDataService, NcjTemplateService } from "app/services";
 import { AutoStorageService } from "app/services/storage";
 import "./market.scss";
 
@@ -25,7 +25,10 @@ export class MarketComponent implements OnInit, OnDestroy {
 
     private _subs: Subscription[] = [];
 
-    constructor(public autoStorageService: AutoStorageService, private templateService: NcjTemplateService) {
+    constructor(
+        public githubDataService: GithubDataService,
+        public autoStorageService: AutoStorageService,
+        private templateService: NcjTemplateService) {
         this._subs.push(this.quicksearch.valueChanges.subscribe((query) => {
             this.query = query;
             this._filterApplications();
@@ -46,7 +49,7 @@ export class MarketComponent implements OnInit, OnDestroy {
 
     @autobind()
     public refreshApplications() {
-        const obs = this.templateService.reloadData().flatMap(() => {
+        const obs = this.githubDataService.reloadData().flatMap(() => {
             return this.templateService.listApplications();
         });
 

app/components/settings/default-settings.json

@@ -21,5 +21,9 @@
   // Default powershell on windows, terminal.app on darwin, ...
   "terminal.external": "default",
 
-  "node.connect.default-username": null
+  // Username to be used by default when connecting to a node(Or adding a user)
+  "node.connect.default-username": null,
+
+  // Branch to be used when loading data from github(This is mostly for experimenting with new NCJ templates)
+  "github-data.source.branch": "master"
 }

app/models/settings.ts

@@ -12,4 +12,5 @@ export interface Settings {
     "auto-update-on-quit": boolean;
     "terminal.external": string;
     "node-connect.default-username": string;
+    "github-data.source.branch": string;
 }

app/services/github-data/github-data.service.spec.ts

@@ -0,0 +1,91 @@
+import { HttpClient } from "@angular/common/http";
+import { HttpClientTestingModule, HttpTestingController } from "@angular/common/http/testing";
+import { TestBed, fakeAsync, tick } from "@angular/core/testing";
+import * as path from "path";
+import { BehaviorSubject } from "rxjs";
+import { GithubDataService } from "./github-data.service";
+
+describe("GithubDataService", () => {
+    let githubDataService: GithubDataService;
+    let httpMock: HttpTestingController;
+    let fsSpy;
+    let settingsSpy;
+
+    beforeEach(() => {
+        settingsSpy = {
+            settingsObs: new BehaviorSubject({ "github-data.source.branch": "master" }),
+        };
+
+        fsSpy = {
+            exists: () => Promise.resolve(false),
+            commonFolders: { temp: "path/to/temp" },
+            readFile: jasmine.createSpy("readFile"),
+            download: jasmine.createSpy("download"),
+            unzip: jasmine.createSpy("unzip"),
+            saveFile: jasmine.createSpy("saveFile"),
+        };
+        TestBed.configureTestingModule({
+            imports: [
+                HttpClientTestingModule,
+            ],
+        });
+        githubDataService = new GithubDataService(TestBed.get(HttpClient), fsSpy, settingsSpy);
+        httpMock = TestBed.get(HttpTestingController);
+    });
+
+    it("is not ready until settings are loaded", fakeAsync(() => {
+        settingsSpy.settingsObs.next({
+            "github-data.source.branch": null,
+        });
+        const readySpy = jasmine.createSpy("ready");
+        githubDataService.ready.subscribe(readySpy);
+        githubDataService.init();
+
+        tick();
+        expect(readySpy).not.toHaveBeenCalled();
+        settingsSpy.settingsObs.next({
+            "github-data.source.branch": "master",
+        });
+
+        tick();
+        expect(readySpy).toHaveBeenCalledOnce();
+    }));
+
+    it("download, unzip and save sync settings", (done) => {
+        githubDataService.init();
+        const zipFile = path.join("path/to/temp", "batch-labs-data.zip");
+        const downloadDir = path.join("path/to/temp", "batch-labs-data");
+        githubDataService.ready.subscribe(() => {
+            expect(fsSpy.download).toHaveBeenCalledOnce();
+            expect(fsSpy.download).toHaveBeenCalledWith(
+                "https://github.com/Azure/BatchLabs-data/archive/master.zip",
+                zipFile);
+            expect(fsSpy.unzip).toHaveBeenCalledOnce();
+            expect(fsSpy.unzip).toHaveBeenCalledWith(
+                zipFile,
+                downloadDir);
+            expect(fsSpy.saveFile).toHaveBeenCalledOnce();
+            expect(fsSpy.saveFile).toHaveBeenCalledWith(
+                path.join(downloadDir, "sync.json"),
+                jasmine.anything(),
+            );
+            const args = fsSpy.saveFile.calls.mostRecent().args;
+            const data = JSON.parse(args[1]);
+            expect(data.source).toEqual("https://github.com/Azure/BatchLabs-data/archive/master.zip");
+            expect(isNaN(Date.parse(data.lastSync))).toBe(false);
+            done();
+        });
+    });
+
+    it("#get gets remote file", (done) => {
+        githubDataService.init();
+        githubDataService.get("some/file/on/github.json").subscribe((result) => {
+            expect(result).toEqual(`{some: "content"}`);
+            done();
+        });
+
+        const response = httpMock.expectOne(
+            "https://raw.githubusercontent.com/Azure/BatchLabs-data/master/some/file/on/github.json");
+        response.flush(`{some: "content"}`);
+    });
+});

app/services/github-data/github-data.service.ts

@@ -0,0 +1,145 @@
+import { HttpClient } from "@angular/common/http";
+import { Injectable, OnDestroy } from "@angular/core";
+import { log } from "@batch-flask/utils";
+import { Constants, DateUtils } from "app/utils";
+import * as path from "path";
+import { AsyncSubject, Observable, Subscription } from "rxjs";
+import { flatMap, share } from "rxjs/operators";
+import { FileSystemService } from "../fs.service";
+import { SettingsService } from "../settings.service";
+
+const repo = "BatchLabs-data";
+const cacheTime = 1; // In days
+
+interface SyncFile {
+    lastSync: Date;
+    source: string;
+}
+
+@Injectable()
+export class GithubDataService implements OnDestroy {
+    public ready: Observable<any>;
+    private _ready = new AsyncSubject();
+
+    private _branch = null;
+    private _settingsSub: Subscription;
+    private _settingsLoaded: Observable<any>;
+
+    constructor(
+        private http: HttpClient,
+        private fs: FileSystemService,
+        private settingsService: SettingsService) {
+        this.ready = this._ready.asObservable();
+    }
+
+    public init() {
+        const obs = this.settingsService.settingsObs;
+        this._settingsLoaded = obs.take(1);
+        this._settingsSub = obs.subscribe((settings) => {
+            const branch = settings["github-data.source.branch"];
+            if (!branch || branch === this._branch) { return; }
+            this._branch = branch;
+            this._updateLocalData();
+        });
+    }
+
+    public reloadData(): Observable<any> {
+        this._ready = new AsyncSubject();
+        return Observable.fromPromise(this._downloadRepo());
+    }
+
+    public ngOnDestroy() {
+        this._settingsSub.unsubscribe();
+    }
+
+    /**
+     * Get the content of the file in github
+     * @param path path relative to the root of the repo
+     */
+    public get(path: string): Observable<string> {
+        return this._settingsLoaded.pipe(
+            flatMap(() => this.http.get(this.getUrl(path), { observe: "body", responseType: "text" })),
+            share(),
+        );
+    }
+
+    /**
+     * Get the remote url for the file
+     * @param path path relative to the root of the repo
+     */
+    public getUrl(path: string): string {
+        return `${this._repoUrl}/${path}`;
+    }
+
+    public getLocalPath(uri: string) {
+        return path.join(this._dataRoot, uri);
+    }
+
+    private get _repoUrl() {
+        return `${Constants.ServiceUrl.githubRaw}/Azure/${repo}/${this._branch}`;
+    }
+
+    private async _checkIfDataNeedReload(): Promise<boolean> {
+        const syncFile = this._syncFile;
+        const exists = await this.fs.exists(syncFile);
+        if (!exists) {
+            return true;
+        }
+        const content = await this.fs.readFile(syncFile);
+        try {
+            const json: SyncFile = JSON.parse(content);
+            const lastSync = new Date(json.lastSync);
+            return json.source !== this._zipUrl || !DateUtils.withinRange(lastSync, cacheTime, "day");
+        } catch (e) {
+            log.error("Error reading sync file. Reloading data from github.", e);
+            return Promise.resolve(true);
+        }
+    }
+
+    private async _downloadRepo() {
+        const tmpZip = path.join(this.fs.commonFolders.temp, "batch-labs-data.zip");
+        const dest = this._repoDownloadRoot;
+        await this.fs.download(this._zipUrl, tmpZip);
+        await this.fs.unzip(tmpZip, dest);
+        await this._saveSyncData(this._zipUrl);
+
+        this._ready.next(true);
+        this._ready.complete();
+    }
+
+    private _saveSyncData(source: string): Promise<string> {
+        const syncFile = this._syncFile;
+        const data: SyncFile = {
+            source,
+            lastSync: new Date(),
+        };
+        const content = JSON.stringify(data);
+        return this.fs.saveFile(syncFile, content);
+    }
+
+    private get _repoDownloadRoot() {
+        return path.join(this.fs.commonFolders.temp, "batch-labs-data");
+    }
+
+    private get _dataRoot() {
+        return path.join(this._repoDownloadRoot, `${repo}-${this._branch}`, "ncj");
+    }
+
+    private get _zipUrl() {
+        return `https://github.com/Azure/${repo}/archive/${this._branch}.zip`;
+    }
+
+    private get _syncFile() {
+        return path.join(this._repoDownloadRoot, "sync.json");
+    }
+
+    private async _updateLocalData() {
+        const needReload = await this._checkIfDataNeedReload();
+        if (!needReload) {
+            this._ready.next(true);
+            this._ready.complete();
+            return null;
+        }
+        await this._downloadRepo();
+    }
+}

app/services/github-data/index.ts

@@ -0,0 +1 @@
+export * from "./github-data.service";

app/services/index.ts

@@ -10,7 +10,7 @@ export * from "./batch-labs.service";
 export * from "./cache-data.service";
 export * from "./compute.service";
 export * from "./fs.service";
-export * from "./github-data.service";
+export * from "./github-data";
 export * from "./http-upload-service";
 export * from "./local-file-storage.service";
 export * from "./monitoring";