Check unverified packages after start/publishing

.npmignore

@@ -7,3 +7,4 @@ appveyor.yml
 Gruntfile.js
 tasks
 test
+src

src/registry/domain/components-cache/components-list.ts

@@ -1,8 +1,9 @@
 import semver from 'semver';
 import pLimit from 'p-limit';
 import getUnixUTCTimestamp from 'oc-get-unix-utc-timestamp';
-import { ComponentsList, Config } from '../../../types';
 import { StorageAdapter } from 'oc-storage-adapters-utils';
+import eventsHandler from '../events-handler';
+import { ComponentsList, Config } from '../../../types';
 
 export default function componentsList(conf: Config, cdn: StorageAdapter) {
   const filePath = (): string =>
@@ -11,17 +12,64 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) {
   const componentsList = {
     getFromJson: (): Promise<ComponentsList> => cdn.getJson(filePath(), true),
 
-    getFromDirectories: async (): Promise<ComponentsList> => {
+    getFromDirectories: async (
+      jsonList: ComponentsList | null
+    ): Promise<ComponentsList> => {
       const componentsInfo: Record<string, string[]> = {};
 
+      const validateComponentVersion = (
+        componentName: string,
+        componentVersion: string
+      ) => {
+        return cdn
+          .getJson(
+            // Check integrity of the package by checking existence of package.json
+            // OC will upload always the package.json last when publishing
+            `${conf.storage.options.componentsDir}/${componentName}/${componentVersion}/package.json`
+          )
+          .then(() => true)
+          .catch(() => false);
+      };
+
       const getVersionsForComponent = async (
         componentName: string
       ): Promise<string[]> => {
-        const versions = await cdn.listSubDirectories(
+        const allVersions = await cdn.listSubDirectories(
           `${conf.storage.options.componentsDir}/${componentName}`
         );
+        const unCheckedVersions = allVersions.filter(
+          version => !jsonList?.components[componentName]?.includes(version)
+        );
+        const limit = pLimit(cdn.maxConcurrentRequests);
+        const invalidVersions = (
+          await Promise.all(
+            unCheckedVersions.map(unCheckedVersion =>
+              limit(async () => {
+                const isValid = await validateComponentVersion(
+                  componentName,
+                  unCheckedVersion
+                );
+
+                return isValid ? null : unCheckedVersion;
+              })
+            )
+          )
+        ).filter((x): x is string => typeof x === 'string');
+
+        if (invalidVersions.length > 0) {
+          eventsHandler.fire('error', {
+            code: 'corrupted_version',
+            message: `Couldn't validate the integrity of the component ${componentName} on the following versions: ${invalidVersions.join(
+              ', '
+            )}.`
+          });
+        }
+
+        const validVersions = allVersions.filter(
+          version => !invalidVersions.includes(version)
+        );
 
-        return versions.sort(semver.compare);
+        return validVersions.sort(semver.compare);
       };
 
       try {
@@ -55,8 +103,8 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) {
       }
     },
 
-    async refresh(): Promise<ComponentsList> {
-      const components = await componentsList.getFromDirectories();
+    async refresh(cachedList: ComponentsList): Promise<ComponentsList> {
+      const components = await componentsList.getFromDirectories(cachedList);
       await componentsList.save(components);
 
       return components;

src/registry/domain/components-cache/index.ts

@@ -3,16 +3,16 @@ import getComponentsList from './components-list';
 import eventsHandler from '../events-handler';
 import getUnixUTCTimestamp from 'oc-get-unix-utc-timestamp';
 import { ComponentsList, Config } from '../../../types';
-import { StorageAdapter } from 'oc-storage-adapters-utils';
+import { StorageAdapter, strings } from 'oc-storage-adapters-utils';
 
 export default function componentsCache(conf: Config, cdn: StorageAdapter) {
   let cachedComponentsList: ComponentsList;
   let refreshLoop: NodeJS.Timeout;
 
   const componentsList = getComponentsList(conf, cdn);
 
-  const poll = () =>
-    setTimeout(async () => {
+  const poll = () => {
+    return setTimeout(async () => {
       try {
         const data = await componentsList.getFromJson();
 
@@ -29,6 +29,7 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
       }
       refreshLoop = poll();
     }, conf.pollingInterval * 1000);
+  };
 
   const cacheDataAndStartPolling = (data: ComponentsList) => {
     cachedComponentsList = data;
@@ -55,12 +56,15 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
     },
 
     async load(): Promise<ComponentsList> {
+      const jsonComponents = await componentsList.getFromJson().catch(err => {
+        if (err?.code === strings.errors.STORAGE.FILE_NOT_FOUND_CODE)
+          return null;
+
+        return Promise.reject(err);
+      });
       const dirComponents = await componentsList
-        .getFromDirectories()
+        .getFromDirectories(jsonComponents)
         .catch(err => throwError('components_list_get', err));
-      const jsonComponents = await componentsList
-        .getFromJson()
-        .catch(() => null);
 
       if (
         !jsonComponents ||
@@ -78,7 +82,8 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
     async refresh(): Promise<ComponentsList> {
       clearTimeout(refreshLoop);
       try {
-        const components = await componentsList.refresh();
+        // Passing components that we know are fine, so it doesn't refresh invalid components
+        const components = await componentsList.refresh(cachedComponentsList);
         cacheDataAndStartPolling(components);
 
         return components;

test/unit/registry-domain-components-cache.js

@@ -45,7 +45,8 @@ describe('registry : domain : components-cache', () => {
         './components-list': injectr(
           '../../dist/registry/domain/components-cache/components-list.js',
           {
-            'oc-get-unix-utc-timestamp': getTimestamp
+            'oc-get-unix-utc-timestamp': getTimestamp,
+            '../events-handler': eventsHandlerStub
           }
         ).default
       },
@@ -59,10 +60,27 @@ describe('registry : domain : components-cache', () => {
   };
 
   describe('when library does not contain components.json', () => {
+    describe('when getting the json fails', () => {
+      let error;
+      before(done => {
+        mockedCdn.getJson = sinon.stub();
+        mockedCdn.getJson.rejects(new Error('FILE_ERROR'));
+        initialise();
+        componentsCache
+          .load()
+          .catch(err => (error = err))
+          .finally(done);
+      });
+
+      it('should throw with the error message', () => {
+        expect(error.message).to.equal('FILE_ERROR');
+      });
+    });
     describe('when initialising the cache', () => {
       before(done => {
         mockedCdn.getJson = sinon.stub();
-        mockedCdn.getJson.rejects('not_found');
+        mockedCdn.getJson.resolves({});
+        mockedCdn.getJson.onFirstCall(0).rejects({ code: 'file_not_found' });
         mockedCdn.listSubDirectories = sinon.stub();
         mockedCdn.listSubDirectories.onCall(0).resolves(['hello-world']);
         mockedCdn.listSubDirectories.onCall(1).resolves(['1.0.0', '1.0.2']);
@@ -72,11 +90,17 @@ describe('registry : domain : components-cache', () => {
         componentsCache.load().finally(done);
       });
 
-      it('should try fetching the components.json', () => {
-        expect(mockedCdn.getJson.calledOnce).to.be.true;
+      it('should try fetching the components.json and check components', () => {
+        expect(mockedCdn.getJson.calledThrice).to.be.true;
         expect(mockedCdn.getJson.args[0][0]).to.be.equal(
           'component/components.json'
         );
+        expect(mockedCdn.getJson.args[1][0]).to.be.equal(
+          'component/hello-world/1.0.0/package.json'
+        );
+        expect(mockedCdn.getJson.args[2][0]).to.be.equal(
+          'component/hello-world/1.0.2/package.json'
+        );
       });
 
       it('should scan for directories to fetch components and versions', () => {
@@ -106,24 +130,48 @@ describe('registry : domain : components-cache', () => {
       before(done => {
         mockedCdn.getJson = sinon.stub();
         mockedCdn.getJson.resolves(baseResponse());
+        mockedCdn.getJson
+          .withArgs('component/hello-world/3.0.0/package.json')
+          .rejects('ERROR');
         mockedCdn.listSubDirectories = sinon.stub();
         mockedCdn.listSubDirectories.onCall(0).resolves(['hello-world']);
         mockedCdn.listSubDirectories
           .onCall(1)
-          .resolves(['1.0.0', '1.0.2', '2.0.0']);
+          .resolves(['1.0.0', '1.0.2', '2.0.0', '3.0.0']);
         mockedCdn.putFileContent = sinon.stub();
         mockedCdn.putFileContent.resolves('ok');
         initialise();
         componentsCache.load().finally(done);
       });
 
       it('should fetch the components.json', () => {
-        expect(mockedCdn.getJson.calledOnce).to.be.true;
         expect(mockedCdn.getJson.args[0][0]).to.be.equal(
           'component/components.json'
         );
       });
 
+      it('should verify new versions', () => {
+        expect(mockedCdn.getJson.calledThrice).to.be.true;
+        expect(mockedCdn.getJson.args[1][0]).to.be.equal(
+          'component/hello-world/2.0.0/package.json'
+        );
+        expect(mockedCdn.getJson.args[2][0]).to.be.equal(
+          'component/hello-world/3.0.0/package.json'
+        );
+      });
+
+      it('should ignore corrupted versions and generate an error event', () => {
+        expect(eventsHandlerStub.fire.called).to.be.true;
+        expect(eventsHandlerStub.fire.args[0][0]).to.equal('error');
+        expect(eventsHandlerStub.fire.args[0][1].code).to.equal(
+          'corrupted_version'
+        );
+        expect(eventsHandlerStub.fire.args[0][1].message).to.contain(
+          'hello-world'
+        );
+        expect(eventsHandlerStub.fire.args[0][1].message).to.contain('3.0.0');
+      });
+
       it('should scan for directories to fetch components and versions', () => {
         expect(mockedCdn.listSubDirectories.calledTwice).to.be.true;
       });

tsconfig.json

@@ -13,7 +13,7 @@
     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
 
     /* Language and Environment */
-    "target": "ES2018" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    "target": "ES2020" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
     // "experimentalDecorators": true,                   /* Enable experimental support for TC39 stage 2 draft decorators. */