fetch list of popular archetypes on the fly (#63)

* fetch list of popular archetypes on the fly * change wording * use constant for URL
microsoft · Jun 8, 2018 · 5a27baa · 5a27baa
1 parent 6e777c9
commit 5a27baa
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 43 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/resources/archetypes.json b/resources/archetypes.json
diff --git a/src/ArchetypeModule.ts b/src/ArchetypeModule.ts
@@ -10,7 +10,7 @@ import { Utils } from "./Utils";
 import { VSCodeUI } from "./VSCodeUI";
 // tslint:disable-next-line:no-http-string
 const REMOTE_ARCHETYPE_CATALOG_URL: string = "http://repo.maven.apache.org/maven2/archetype-catalog.xml";
-
+const POPULAR_ARCHETYPES_URL: string = "https://vscodemaventelemetry.blob.core.windows.net/public/popular_archetypes.json";
 class Step {
     public readonly name: string;
     public readonly info: string;
@@ -50,7 +50,7 @@ export namespace ArchetypeModule {
     }
 
     export async function updateArchetypeCatalog(): Promise<void> {
-        const xml: string = await Utils.httpGetContent(REMOTE_ARCHETYPE_CATALOG_URL);
+        const xml: string = await Utils.downloadFile(REMOTE_ARCHETYPE_CATALOG_URL, true);
         const archetypes: Archetype[] = await Utils.listArchetypeFromXml(xml);
         const targetFilePath: string = path.join(Utils.getPathToExtensionRoot(), "resources", "archetypes.json");
         await fs.ensureFile(targetFilePath);
@@ -63,7 +63,7 @@ export namespace ArchetypeModule {
             (item: Archetype) => item.artifactId ? `$(package) ${item.artifactId} ` : "More ...",
             (item: Archetype) => item.groupId ? `${item.groupId}` : "",
             (item: Archetype) => item.description,
-            { matchOnDescription: true, placeHolder: "Select archetype with <groupId>:<artifactId> ..." }
+            { matchOnDescription: true, placeHolder: "Select an archetype ..." }
         );
     }
 
@@ -110,21 +110,25 @@ export namespace ArchetypeModule {
     }
 
     async function getRecomendedItems(allItems: Archetype[]): Promise<Archetype[]> {
-        // tslint:disable-next-line:no-suspicious-comment
-        // TODO: should not hard code.
-        // Top 10 popular archetypes according to usage data
-        const fixedList: string[] = [
-            "org.apache.maven.archetypes:maven-archetype-quickstart",
-            "org.apache.maven.archetypes:maven-archetype-archetype",
-            "org.apache.maven.archetypes:maven-archetype-webapp",
-            "org.apache.maven.archetypes:maven-archetype-j2ee-simple",
-            "com.microsoft.azure:azure-functions-archetype",
-            "am.ik.archetype:maven-reactjs-blank-archetype",
-            "com.microsoft.azure.gateway.archetypes:gateway-module-simple",
-            "org.apache.maven.archetypes:maven-archetype-site-simple",
-            "com.github.ngeor:archetype-quickstart-jdk8",
-            "org.apache.maven.archetypes:maven-archetype-plugin"
-        ];
+        // Top popular archetypes according to usage data
+        let fixedList: string[];
+        try {
+            const rawlist: string = await Utils.downloadFile(POPULAR_ARCHETYPES_URL, true);
+            fixedList = JSON.parse(rawlist);
+        } catch (error) {
+            fixedList = [
+                "org.apache.maven.archetypes:maven-archetype-quickstart",
+                "org.apache.maven.archetypes:maven-archetype-archetype",
+                "org.apache.maven.archetypes:maven-archetype-webapp",
+                "org.apache.maven.archetypes:maven-archetype-j2ee-simple",
+                "com.microsoft.azure:azure-functions-archetype",
+                "am.ik.archetype:maven-reactjs-blank-archetype",
+                "com.microsoft.azure.gateway.archetypes:gateway-module-simple",
+                "org.apache.maven.archetypes:maven-archetype-site-simple",
+                "com.github.ngeor:archetype-quickstart-jdk8",
+                "org.apache.maven.archetypes:maven-archetype-plugin"
+            ];
+        }
         return fixedList.map((fullname: string) => allItems.find((item: Archetype) => fullname === `${item.groupId}:${item.artifactId}`));
     }
 }
diff --git a/src/Utils.ts b/src/Utils.ts
@@ -3,10 +3,12 @@
 
 import * as fse from "fs-extra";
 import * as http from "http";
+import * as https from "https";
 import * as md5 from "md5";
 import * as minimatch from "minimatch";
 import * as os from "os";
 import * as path from "path";
+import * as url from "url";
 import { ExtensionContext, extensions, Uri, workspace } from 'vscode';
 import * as xml2js from "xml2js";
 import { Archetype } from "./model/Archetype";
@@ -52,6 +54,10 @@ export namespace Utils {
         return path.join(os.tmpdir(), EXTENSION_NAME, ...args);
     }
 
+    export function getTempFolder(): string {
+        return path.join(os.tmpdir(), getExtensionId());
+    }
+
     export function getPathToExtensionRoot(...args: string[]): string {
         return path.join(extensions.getExtension(getExtensionId()).extensionPath, ...args);
     }
@@ -161,27 +167,52 @@ export namespace Utils {
         return path.join(Utils.getPathToExtensionRoot(), "resources", "archetype-catalog.xml");
     }
 
-    export async function httpGetContent(url: string): Promise<string> {
-        const filepath: string = getTempFolderPath(md5(url));
-        if (await fse.pathExists(filepath)) {
-            await fse.unlink(filepath);
+    export async function downloadFile(targetUrl: string, readContent?: boolean, customHeaders?: {}): Promise<string> {
+        const tempFilePath: string = path.join(getTempFolder(), md5(targetUrl));
+        await fse.ensureDir(getTempFolder());
+        if (await fse.pathExists(tempFilePath)) {
+            await fse.remove(tempFilePath);
         }
-        await fse.ensureFile(filepath);
-        const file: fse.WriteStream = fse.createWriteStream(filepath);
-        return new Promise<string>(
-            (resolve: (value: string) => void, reject: (e: Error) => void): void => {
-                const request: http.ClientRequest = http.get(url, (response: http.IncomingMessage) => {
-                    response.pipe(file);
-                    file.on('finish', async () => {
-                        file.close();
-                        const buf: Buffer = await fse.readFile(filepath);
-                        resolve(buf.toString());
-                    });
+
+        return await new Promise((resolve: (res: string) => void, reject: (e: Error) => void): void => {
+            const urlObj: url.Url = url.parse(targetUrl);
+            const options: Object = Object.assign({ headers: Object.assign({}, customHeaders, { 'User-Agent': `vscode/${getExtensionVersion()}` }) }, urlObj);
+            let client: any;
+            if (urlObj.protocol === "https:") {
+                client = https;
+                // tslint:disable-next-line:no-http-string
+            } else if (urlObj.protocol === "http:") {
+                client = http;
+            } else {
+                return reject(new Error("Unsupported protocol."));
+            }
+            client.get(options, (res: http.IncomingMessage) => {
+                let rawData: string;
+                let ws: fse.WriteStream;
+                if (readContent) {
+                    rawData = "";
+                } else {
+                    ws = fse.createWriteStream(tempFilePath);
+                }
+                res.on('data', (chunk: string | Buffer) => {
+                    if (readContent) {
+                        rawData += chunk;
+                    } else {
+                        ws.write(chunk);
+                    }
                 });
-                request.on("error", (e: Error) => {
-                    reject(e);
+                res.on('end', () => {
+                    if (readContent) {
+                        resolve(rawData);
+                    } else {
+                        ws.end();
+                        resolve(tempFilePath);
+                    }
                 });
+            }).on("error", (err: Error) => {
+                reject(err);
             });
+        });
     }
 
     export async function findAllInDir(currentPath: string, targetFileName: string, depth: number, exclusion: string[] = ["**/.*"]): Promise<string[]> {