Add fetch for categories json on validation

__tests__/lib/polling/validator.spec.ts

@@ -1,104 +1,124 @@
-import { validateText } from 'modules/polling/helpers/validator';
+import { validateText, hardcodedCategories } from 'modules/polling/helpers/validator';
 import fs from 'fs';
 const pollMetadata = fs.readFileSync(__dirname + '/poll-431.md').toString();
 
 test('accept a valid document', () => {
-  const result = validateText(pollMetadata);
+  const result = validateText(pollMetadata, hardcodedCategories);
   expect(result.valid).toBeTruthy();
   expect(result.errors.length).toBe(0);
 });
 
 test('reject a blank document', () => {
-  const result = validateText('');
+  const result = validateText('', hardcodedCategories);
   expect(result.valid).toBeFalsy();
 });
 
 test('reject null', () => {
-  const result = validateText(null);
+  const result = validateText(null, hardcodedCategories);
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('expected input to be a string or buffer');
 });
 
 test('reject a document with no options', () => {
-  const result = validateText(`---
+  const result = validateText(
+    `---
 vote_type: Plurality Voting
 ---
 # Hello world
-  `);
+  `,
+    hardcodedCategories
+  );
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('Vote options are missing');
 });
 
 test('reject a document with bad options type', () => {
-  const result = validateText(`---
+  const result = validateText(
+    `---
 vote_type: Plurality Voting
 options: wat
 ---
 # Hello world
-    `);
+    `,
+    hardcodedCategories
+  );
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('Vote options must be a numbered list');
 });
 
 test('reject a document with bad options keys', () => {
-  const result = validateText(`---
+  const result = validateText(
+    `---
 vote_type: Plurality Voting
 options:
   0: foo
   1: bar
   three: baz
 ---
 # Hello world
-    `);
+    `,
+    hardcodedCategories
+  );
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('Vote option IDs must be numbers');
 });
 
 test('reject a document with invalid vote type', () => {
-  const result = validateText(pollMetadata.replace('Plurality', 'Blurality'));
+  const result = validateText(pollMetadata.replace('Plurality', 'Blurality'), hardcodedCategories);
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('Invalid vote type: "Blurality Voting"');
 });
 
 test('reject a document with no categories', () => {
-  const result = validateText(`---
+  const result = validateText(
+    `---
 vote_type: Plurality Voting
 options:
   0: foo
   1: bar
 
 ---
-# Hello world`);
+# Hello world`,
+    hardcodedCategories
+  );
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('Categories are missing');
 });
 
 test('reject a document with invalid category', () => {
   const result = validateText(
-    pollMetadata.replace('Technical', 'Jechnical').replace('Risk Variable', 'Zisk Zariable')
+    pollMetadata.replace('Technical', 'Jechnical').replace('Risk Variable', 'Zisk Zariable'),
+    hardcodedCategories
   );
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('Invalid categories: Zisk Zariable, Jechnical');
 });
 
 test('reject a document with a missing date', () => {
-  const result = validateText(pollMetadata.replace('start_date', 'x').replace('end_date', 'y'));
+  const result = validateText(
+    pollMetadata.replace('start_date', 'x').replace('end_date', 'y'),
+    hardcodedCategories
+  );
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('Start date is missing');
   expect(result.errors).toContain('End date is missing');
 });
 
 test('reject a document with an invalid date', () => {
   const result = validateText(
-    pollMetadata.replace(/start_date: .*/, 'start_date: foo').replace(/end_date: .*/, 'end_date: bar')
+    pollMetadata.replace(/start_date: .*/, 'start_date: foo').replace(/end_date: .*/, 'end_date: bar'),
+    hardcodedCategories
   );
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('Invalid start date (should be like "2020-08-16T08:00:00")');
   expect(result.errors).toContain('Invalid end date (should be like "2020-08-16T08:00:00")');
 });
 
 test('reject a document with an invalid duration', () => {
-  const result = validateText(pollMetadata.replace(/end_date: .*/, 'end_date: 2021-01-18T16:59:00'));
+  const result = validateText(
+    pollMetadata.replace(/end_date: .*/, 'end_date: 2021-01-18T16:59:00'),
+    hardcodedCategories
+  );
   expect(result.valid).toBeFalsy();
   expect(result.errors).toContain('Poll duration is too short');
 });

modules/polling/helpers/validator.ts

@@ -7,7 +7,7 @@ import { POLL_VOTE_TYPES_ARRAY } from '../polling.constants';
 
 // find the most up-to-date list here:
 // https://github.com/makerdao/community/blob/master/governance/polls/meta/categories.json
-const validCategories = [
+export const hardcodedCategories = [
   'Collateral',
   'Oracles',
   'Governance',
@@ -32,18 +32,31 @@ type ValidationResult = {
   wholeDoc?: string;
 };
 
+export async function fetchCategories(): Promise<string[]> {
+  try {
+    const url =
+      'https://raw.githubusercontent.com/makerdao/community/master/governance/polls/meta/categories.json';
+    const resp = await fetch(url);
+    return resp.json();
+  } catch (err) {
+    // fallback to hardcoded categories if live category fetch fails
+    return hardcodedCategories;
+  }
+}
+
 export async function validateUrl(url: string, poll?: PartialPoll): Promise<ValidationResult> {
   const resp = await fetch(url);
   const text = await resp.text();
-  const result = validateText(text);
+  const categories = await fetchCategories();
+  const result = validateText(text, categories);
   if (result.valid && poll) {
     result.wholeDoc = text;
     result.parsedData = parsePollMetadata(poll, text);
   }
   return result;
 }
 
-export function validateText(text: string): ValidationResult {
+export function validateText(text: string, categories: string[]): ValidationResult {
   try {
     const { data, content } = matter(text);
     if (!content) return { valid: false, errors: ['Document is blank'] };
@@ -74,7 +87,7 @@ export function validateText(text: string): ValidationResult {
     if (!data.categories || isEmpty(data.categories)) {
       errors.push('Categories are missing');
     } else {
-      const invalidCategories = difference(data.categories, validCategories);
+      const invalidCategories = difference(data.categories, categories);
       if (invalidCategories.length > 0) errors.push(`Invalid categories: ${invalidCategories.join(', ')}`);
     }
 

yarn.lock

@@ -13570,7 +13570,7 @@ web3-net@1.6.0:
     web3-core-method "1.6.0"
     web3-utils "1.6.0"
 
-web3-provider-engine@14.0.4, "web3-provider-engine@github:makerdao/provider-engine#kovan-fix-dist", web3-provider-engine@makerdao/provider-engine#kovan-fix-dist:
+web3-provider-engine@14.0.4, web3-provider-engine@makerdao/provider-engine#kovan-fix-dist:
   version "14.0.4"
   resolved "https://codeload.github.com/makerdao/provider-engine/tar.gz/7c4b187809f156409473246de15dc595e1fe3356"
   dependencies: