Merge branch 'main' into impl-put-multipart

bindings/nodejs/CONTRIBUTING.md

@@ -54,6 +54,27 @@ To verify that everything is working properly, run `pnpm --version`:
 8.11.0
 ```
 
+### Using [devbox](https://www.jetify.com/devbox/docs/)
+
+Launch Development Environment
+
+```shell
+devbox shell
+```
+
+Contains the following tools:
+
+- `node`: `18`
+- `pnpm`: `8.14.0`
+- `libiconv`: fix nix `ld` missing issue
+
+[Running Scripts](https://www.jetify.com/devbox/docs/guides/scripts/)
+
+- `devbox run format`
+- `devbox run test`
+- `devbox run build`
+- `devbox run dev`
+
 ## Build
 
 ```bash

bindings/nodejs/devbox.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.11.1/.schema/devbox.schema.json",
+  "packages": ["nodejs@18", "nodePackages.pnpm@8.14.0", "libiconv"],
+  "env": {
+    "OPENDAL_TEST": "memory"
+  },
+  "shell": {
+    "init_hook": ["pnpm install"],
+    "scripts": {
+      "format": "pnpm format && cargo fmt",
+      "test": "pnpm test",
+      "build": "pnpm build",
+      "dev": "pnpm build:debug"
+    }
+  }
+}

bindings/nodejs/devbox.lock

@@ -0,0 +1,132 @@
+{
+  "lockfile_version": "1",
+  "packages": {
+    "libiconv": {
+      "resolved": "github:NixOS/nixpkgs/75a52265bda7fd25e06e3a67dee3f0354e73243c#libiconv",
+      "source": "nixpkg",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "path": "/nix/store/xzgz4yl47qjadcjwqcll08dx930mr324-libiconv-50",
+              "default": true
+            }
+          ]
+        }
+      }
+    },
+    "nodePackages.pnpm@8.14.0": {
+      "last_modified": "2024-01-27T14:55:31Z",
+      "resolved": "github:NixOS/nixpkgs/160b762eda6d139ac10ae081f8f78d640dd523eb#nodePackages.pnpm",
+      "source": "devbox-search",
+      "version": "8.14.0",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/349jy7qf0vf0i5y469hs151yd4wqx6z2-pnpm-8.14.0",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/349jy7qf0vf0i5y469hs151yd4wqx6z2-pnpm-8.14.0"
+        },
+        "aarch64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/0q1j1iw30kq20j3g690knr3bjx2j68qq-pnpm-8.14.0",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/0q1j1iw30kq20j3g690knr3bjx2j68qq-pnpm-8.14.0"
+        },
+        "x86_64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/7p7djg6zpyvbhp727wvfwyivw955f72b-pnpm-8.14.0",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/7p7djg6zpyvbhp727wvfwyivw955f72b-pnpm-8.14.0"
+        },
+        "x86_64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/nccby89wifbry3qan555vvqp2qa6my4l-pnpm-8.14.0",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/nccby89wifbry3qan555vvqp2qa6my4l-pnpm-8.14.0"
+        }
+      }
+    },
+    "nodejs@18": {
+      "last_modified": "2024-05-22T06:18:38Z",
+      "plugin_version": "0.0.2",
+      "resolved": "github:NixOS/nixpkgs/3f316d2a50699a78afe5e77ca486ad553169061e#nodejs_18",
+      "source": "devbox-search",
+      "version": "18.20.2",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/ryz5m9pfjydwh2jn8rnhmbiy1hl1q3qp-nodejs-18.20.2",
+              "default": true
+            },
+            {
+              "name": "libv8",
+              "path": "/nix/store/j2lq9vgcjfq4a1q1vdiq4xyra99nz61c-nodejs-18.20.2-libv8"
+            }
+          ],
+          "store_path": "/nix/store/ryz5m9pfjydwh2jn8rnhmbiy1hl1q3qp-nodejs-18.20.2"
+        },
+        "aarch64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/lpvidxshdc4ibdhqrp1z7p5zg1z40f43-nodejs-18.20.2",
+              "default": true
+            },
+            {
+              "name": "libv8",
+              "path": "/nix/store/yg7inirajpsx6qnl95pwnksmg57zh3q2-nodejs-18.20.2-libv8"
+            }
+          ],
+          "store_path": "/nix/store/lpvidxshdc4ibdhqrp1z7p5zg1z40f43-nodejs-18.20.2"
+        },
+        "x86_64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/iydhsvwc8pmyhzm2zh0lsf8k3x49kbqf-nodejs-18.20.2",
+              "default": true
+            },
+            {
+              "name": "libv8",
+              "path": "/nix/store/7xmwmd7s1mi6l87ypaca4j0ssbmd9a2f-nodejs-18.20.2-libv8"
+            }
+          ],
+          "store_path": "/nix/store/iydhsvwc8pmyhzm2zh0lsf8k3x49kbqf-nodejs-18.20.2"
+        },
+        "x86_64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/l6sykmmiavsfl44p16643x59282n78ll-nodejs-18.20.2",
+              "default": true
+            },
+            {
+              "name": "libv8",
+              "path": "/nix/store/2039ina8wsw34gdylmzlqgmchys5yvhg-nodejs-18.20.2-libv8"
+            }
+          ],
+          "store_path": "/nix/store/l6sykmmiavsfl44p16643x59282n78ll-nodejs-18.20.2"
+        }
+      }
+    }
+  }
+}

core/src/layers/async_backtrace.rs

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use futures::FutureExt;
+
 use crate::raw::*;
 use crate::*;
 
@@ -58,25 +60,31 @@ pub struct AsyncBacktraceAccessor<A: Access> {
 
 impl<A: Access> LayeredAccess for AsyncBacktraceAccessor<A> {
     type Inner = A;
-    type Reader = A::Reader;
-    type BlockingReader = A::BlockingReader;
-    type Writer = A::Writer;
-    type BlockingWriter = A::BlockingWriter;
-    type Lister = A::Lister;
-    type BlockingLister = A::BlockingLister;
+    type Reader = AsyncBacktraceWrapper<A::Reader>;
+    type BlockingReader = AsyncBacktraceWrapper<A::BlockingReader>;
+    type Writer = AsyncBacktraceWrapper<A::Writer>;
+    type BlockingWriter = AsyncBacktraceWrapper<A::BlockingWriter>;
+    type Lister = AsyncBacktraceWrapper<A::Lister>;
+    type BlockingLister = AsyncBacktraceWrapper<A::BlockingLister>;
 
     fn inner(&self) -> &Self::Inner {
         &self.inner
     }
 
     #[async_backtrace::framed]
     async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
-        self.inner.read(path, args).await
+        self.inner
+            .read(path, args)
+            .map(|v| v.map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r))))
+            .await
     }
 
     #[async_backtrace::framed]
     async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
-        self.inner.write(path, args).await
+        self.inner
+            .write(path, args)
+            .map(|v| v.map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r))))
+            .await
     }
 
     #[async_backtrace::framed]
@@ -101,7 +109,10 @@ impl<A: Access> LayeredAccess for AsyncBacktraceAccessor<A> {
 
     #[async_backtrace::framed]
     async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
-        self.inner.list(path, args).await
+        self.inner
+            .list(path, args)
+            .map(|v| v.map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r))))
+            .await
     }
 
     #[async_backtrace::framed]
@@ -115,14 +126,83 @@ impl<A: Access> LayeredAccess for AsyncBacktraceAccessor<A> {
     }
 
     fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
-        self.inner.blocking_read(path, args)
+        self.inner
+            .blocking_read(path, args)
+            .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
     }
 
     fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
-        self.inner.blocking_write(path, args)
+        self.inner
+            .blocking_write(path, args)
+            .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
     }
 
     fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
-        self.inner.blocking_list(path, args)
+        self.inner
+            .blocking_list(path, args)
+            .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
+    }
+}
+
+pub struct AsyncBacktraceWrapper<R> {
+    inner: R,
+}
+
+impl<R> AsyncBacktraceWrapper<R> {
+    fn new(inner: R) -> Self {
+        Self { inner }
+    }
+}
+
+impl<R: oio::Read> oio::Read for AsyncBacktraceWrapper<R> {
+    #[async_backtrace::framed]
+    async fn read(&mut self) -> Result<Buffer> {
+        self.inner.read().await
+    }
+}
+
+impl<R: oio::BlockingRead> oio::BlockingRead for AsyncBacktraceWrapper<R> {
+    fn read(&mut self) -> Result<Buffer> {
+        self.inner.read()
+    }
+}
+
+impl<R: oio::Write> oio::Write for AsyncBacktraceWrapper<R> {
+    #[async_backtrace::framed]
+    async fn write(&mut self, bs: Buffer) -> Result<usize> {
+        self.inner.write(bs).await
+    }
+
+    #[async_backtrace::framed]
+    async fn abort(&mut self) -> Result<()> {
+        self.inner.abort().await
+    }
+
+    #[async_backtrace::framed]
+    async fn close(&mut self) -> Result<()> {
+        self.inner.close().await
+    }
+}
+
+impl<R: oio::BlockingWrite> oio::BlockingWrite for AsyncBacktraceWrapper<R> {
+    fn write(&mut self, bs: Buffer) -> Result<usize> {
+        self.inner.write(bs)
+    }
+
+    fn close(&mut self) -> Result<()> {
+        self.inner.close()
+    }
+}
+
+impl<R: oio::List> oio::List for AsyncBacktraceWrapper<R> {
+    #[async_backtrace::framed]
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        self.inner.next().await
+    }
+}
+
+impl<R: oio::BlockingList> oio::BlockingList for AsyncBacktraceWrapper<R> {
+    fn next(&mut self) -> Result<Option<oio::Entry>> {
+        self.inner.next()
     }
 }