Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: mini fixes #659

Merged
merged 1 commit into from
Jul 12, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/bootstrap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ export async function run(

// Push nothing to trigger event
consumer.push(null as any);
processing = false;
}

consumer.pause();
Expand Down Expand Up @@ -159,7 +158,7 @@ function createPkgConsumer(
try {
datadog.increment('packages');

const res = await npm.getDoc(pkg.id);
const res = await npm.getDoc(pkg.id, pkg.value.rev);

if (isFailure(res)) {
log.error('Got an error', res.error);
Expand Down
9 changes: 7 additions & 2 deletions src/npm/Prefetcher.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
import type { DocumentListParams } from 'nano';
import type { DocumentListParams, DocumentResponseRow } from 'nano';

import { config } from '../config';
import { log } from '../utils/log';
import { wait } from '../utils/wait';

import type { GetPackage } from './types';

import * as npm from './index';

export type PrefetchedPkg = { id: string };
export type PrefetchedPkg = Pick<
DocumentResponseRow<GetPackage>,
'id' | 'value'
>;

export class Prefetcher {
#limit: number = config.bootstrapConcurrency;
Expand Down
7 changes: 5 additions & 2 deletions src/npm/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,13 @@ async function getChanges(
return results;
}

async function getDoc(name: string): Promise<DocumentGetResponse & GetPackage> {
async function getDoc(
name: string,
rev: string
): Promise<DocumentGetResponse & GetPackage> {
const start = Date.now();

const doc = await db.get(name);
const doc = await db.get(name, { rev });

datadog.timing('npm.getDoc', Date.now() - start);

Expand Down
26 changes: 17 additions & 9 deletions src/watch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,19 +112,27 @@ async function loop(
if (change.deleted) {
// Delete package directly in index
// Filter does not support async/await but there is no concurrency issue with this
mainIndex.deleteObject(change.id);
log.info(`Deleted`, change.id);
return;
throw new Error('deleted');
}

const res = await npm.getDoc(change.id);
try {
const res = await npm.getDoc(change.id, change.changes[0].rev);

if (isFailure(res)) {
log.error('Got an error', res.error);
return;
}
if (isFailure(res)) {
log.error('Got an error', res.error);
return;
}

await saveDoc({ row: res, index: mainIndex });
await saveDoc({ row: res, index: mainIndex });
} catch (e) {
// this error can be thrown by us or by nano if:
// - we received a change that is not marked as "deleted"
// - and the package has since been deleted
if (e.message === 'deleted') {
mainIndex.deleteObject(change.id);
log.info(`deleted`, change.id);
}
}
}

/**
Expand Down