Skip to content

Commit

Permalink
Only compute logits hash if we generated a token
Browse files Browse the repository at this point in the history
There are rare cases where the prompt is larger than the number of steps
so we never actually do a full forward pass. Also exclude hash
computation from timing bracket.
  • Loading branch information
zeux committed Jun 7, 2024
1 parent 74eec37 commit 1a7ca47
Showing 1 changed file with 6 additions and 3 deletions.
9 changes: 6 additions & 3 deletions src/run.c
Original file line number Diff line number Diff line change
Expand Up @@ -248,13 +248,16 @@ void generate(struct Transformer* transformer, struct Tokenizer* tokenizer, stru
}
printf("\n");

long end = time_in_ms();

// fold last token's logits into a hash for validation
unsigned logits_hash = 0;
for (int k = 0; k < transformer->config.vocab_size; ++k) {
logits_hash = logits_hash * 5 + *(unsigned*)(&logits_last[k]);
if (logits_last) {
for (int k = 0; k < transformer->config.vocab_size; ++k) {
logits_hash = logits_hash * 5 + *(unsigned*)(&logits_last[k]);
}
}

long end = time_in_ms();
fprintf(stderr, "# %d tokens: throughput: %.2f tok/s; latency: %.2f ms/tok; bandwidth: %.2f GB/s; total %.3f sec; #%08x\n",
pos,
pos / (double)(end - start) * 1000, (double)(end - start) / pos,
Expand Down

0 comments on commit 1a7ca47

Please sign in to comment.