Skip to content

Commit

Permalink
fix: replace fast-atomic-write with steno (#285)
Browse files Browse the repository at this point in the history
fixes #284

fast-write-atomic hasn't been updated in 5 years, is CJS, and is slower than steno (updated 2 months ago).

Benchmarks for various content-types & libraries (though we only use Uint8Arrays) can be found at https://github.com/SgtPooki/fast-write-atomic#benchmarks

However, there may be further room for improvement by moving to [fs.createWriteStream](https://nodejs.org/api/fs.html#fscreatewritestreampath-options)


```
╰─ ✔ ❯ hyperfine --parameter-list branch 284-chore-replace-fast-write-atomic-with-steno,main --setup "git switch {branch} && npm run reset && npm i && npm run build" --runs 20  -w 1 "npm run test:node"
Benchmark 1: npm run test:node (branch = 284-chore-replace-fast-write-atomic-with-steno)
  Time (mean ± σ):     27.212 s ±  0.832 s    [User: 34.810 s, System: 6.051 s]
  Range (min … max):   25.927 s … 29.324 s    20 runs

Benchmark 2: npm run test:node (branch = main)
  Time (mean ± σ):     42.971 s ±  0.637 s    [User: 35.297 s, System: 7.534 s]
  Range (min … max):   42.178 s … 44.796 s    20 runs

Summary
  npm run test:node (branch = 284-chore-replace-fast-write-atomic-with-steno) ran
    1.58 ± 0.05 times faster than npm run test:node (branch = main)
```

---

### Updated benchmarks of `npm run test` as of 2024-04-19

```
╭─    ~/code/work/protocol.ai/ipfs/js-stores    main ?1 
╰─ ✔ ❯ hyperfine --parameter-list branch main,test/not-same-event-loop-concurrency,284-chore-replace-fast-write-atomic-with-steno --setup "git switch {branch} && npm run reset && npm i && npm run build && cd packages/datastore-fs" "npm run test"
Benchmark 1: npm run test (branch = main)
  Time (mean ± σ):     99.415 s ±  2.918 s    [User: 69.659 s, System: 23.361 s]
  Range (min … max):   96.134 s … 105.200 s    10 runs

Benchmark 2: npm run test (branch = test/not-same-event-loop-concurrency)
  Time (mean ± σ):     103.456 s ±  3.186 s    [User: 74.442 s, System: 25.261 s]
  Range (min … max):   98.813 s … 108.429 s    10 runs

Benchmark 3: npm run test (branch = 284-chore-replace-fast-write-atomic-with-steno)
  Time (mean ± σ):     80.308 s ±  2.107 s    [User: 74.331 s, System: 22.228 s]
  Range (min … max):   78.219 s … 84.277 s    10 runs

Summary
  npm run test (branch = 284-chore-replace-fast-write-atomic-with-steno) ran
    1.24 ± 0.05 times faster than npm run test (branch = main)
    1.29 ± 0.05 times faster than npm run test (branch = test/not-same-event-loop-concurrency)                                                                                        [49m1.944s]

╭─    ~/code/work/protocol.ai/ipfs/js-stores    284-chore-re…c-with-steno ?1 
╰─ ✔ ❯ hyperfine --parameter-list branch main,test/not-same-event-loop-concurrency,284-chore-replace-fast-write-atomic-with-steno --setup "git switch {branch} && npm run reset && npm i && npm run build && cd packages/blockstore-fs" "npm run test"
Benchmark 1: npm run test (branch = main)
  Time (mean ± σ):     98.840 s ±  2.612 s    [User: 68.486 s, System: 22.585 s]
  Range (min … max):   97.005 s … 104.396 s    10 runs

Benchmark 2: npm run test (branch = test/not-same-event-loop-concurrency)
  Time (mean ± σ):     105.307 s ±  2.335 s    [User: 72.442 s, System: 24.766 s]
  Range (min … max):   101.167 s … 109.007 s    10 runs

Benchmark 3: npm run test (branch = 284-chore-replace-fast-write-atomic-with-steno)
  Time (mean ± σ):     77.012 s ±  1.829 s    [User: 74.442 s, System: 21.938 s]
  Range (min … max):   75.258 s … 80.825 s    10 runs

Summary
  npm run test (branch = 284-chore-replace-fast-write-atomic-with-steno) ran
    1.28 ± 0.05 times faster than npm run test (branch = main)
    1.37 ± 0.04 times faster than npm run test (branch = test/not-same-event-loop-concurrency)
```
  • Loading branch information
SgtPooki authored Aug 9, 2024
1 parent f435854 commit c6df8dd
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 22 deletions.
4 changes: 2 additions & 2 deletions packages/blockstore-fs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,13 @@
},
"dependencies": {
"blockstore-core": "^4.0.0",
"fast-write-atomic": "^0.2.1",
"interface-blockstore": "^5.0.0",
"interface-store": "^5.0.0",
"it-glob": "^2.0.6",
"it-map": "^3.0.5",
"it-parallel-batch": "^3.0.4",
"multiformats": "^13.0.1"
"multiformats": "^13.0.1",
"steno": "^4.0.2"
},
"devDependencies": {
"aegir": "^42.2.3",
Expand Down
14 changes: 5 additions & 9 deletions packages/blockstore-fs/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,29 @@

import fs from 'node:fs/promises'
import path from 'node:path'
import { promisify } from 'node:util'
import {
Errors
} from 'blockstore-core'
// @ts-expect-error no types
import fwa from 'fast-write-atomic'
import glob from 'it-glob'
import map from 'it-map'
import parallelBatch from 'it-parallel-batch'
import { Writer } from 'steno'
import { NextToLast, type ShardingStrategy } from './sharding.js'
import type { Blockstore, Pair } from 'interface-blockstore'
import type { AwaitIterable } from 'interface-store'
import type { CID } from 'multiformats/cid'

const writeAtomic = promisify(fwa)

/**
* Write a file atomically
*/
async function writeFile (file: string, contents: Uint8Array): Promise<void> {
try {
await writeAtomic(file, contents)
const writer = new Writer(file)
await writer.write(contents)
} catch (err: any) {
if (err.syscall === 'rename' && ['ENOENT', 'EPERM'].includes(err.code)) {
// fast-write-atomic writes a file to a temp location before renaming it.
// On Windows, if the final file already exists this error is thrown.
// No such error is thrown on Linux/Mac
// steno writes a file to a temp location before renaming it.
// If the final file already exists this error is thrown.
// Make sure we can read & write to this file
await fs.access(file, fs.constants.F_OK | fs.constants.W_OK)

Expand Down
4 changes: 2 additions & 2 deletions packages/datastore-fs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,12 @@
},
"dependencies": {
"datastore-core": "^9.0.0",
"fast-write-atomic": "^0.2.1",
"interface-datastore": "^8.0.0",
"interface-store": "^5.0.0",
"it-glob": "^2.0.6",
"it-map": "^3.0.5",
"it-parallel-batch": "^3.0.4"
"it-parallel-batch": "^3.0.4",
"steno": "^4.0.2"
},
"devDependencies": {
"@types/mkdirp": "^2.0.0",
Expand Down
14 changes: 5 additions & 9 deletions packages/datastore-fs/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,29 @@

import fs from 'node:fs/promises'
import path from 'node:path'
import { promisify } from 'util'
import {
BaseDatastore, Errors
} from 'datastore-core'
// @ts-expect-error no types
import fwa from 'fast-write-atomic'
import {
Key, type KeyQuery, type Pair, type Query
} from 'interface-datastore'
import glob from 'it-glob'
import map from 'it-map'
import parallel from 'it-parallel-batch'
import { Writer } from 'steno'
import type { AwaitIterable } from 'interface-store'

const writeAtomic = promisify(fwa)

/**
* Write a file atomically
*/
async function writeFile (path: string, contents: Uint8Array): Promise<void> {
try {
await writeAtomic(path, contents)
const writer = new Writer(path)
await writer.write(contents)
} catch (err: any) {
if (err.syscall === 'rename' && ['ENOENT', 'EPERM'].includes(err.code)) {
// fast-write-atomic writes a file to a temp location before renaming it.
// On Windows, if the final file already exists this error is thrown.
// No such error is thrown on Linux/Mac
// steno writes a file to a temp location before renaming it.
// If the final file already exists this error is thrown.
// Make sure we can read & write to this file
await fs.access(path, fs.constants.F_OK | fs.constants.W_OK)

Expand Down

0 comments on commit c6df8dd

Please sign in to comment.