diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000000000..17743fbd00d7a1 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: denoland diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 902dad72b836ac..532a7c1e744898 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: run: git config --global core.symlinks true - name: Clone repository - uses: actions/checkout@v1 + uses: actions/checkout@v2 with: # Use depth > 1, because sometimes we need to rebuild master and if # other commits have landed it will become impossible to rebuild if @@ -66,7 +66,7 @@ jobs: - name: Install rust uses: hecrj/setup-rust-action@v1 with: - rust-version: "1.43.0" + rust-version: 1.44.0 - name: Install clippy and rustfmt if: matrix.config.kind == 'lint' @@ -104,7 +104,7 @@ jobs: run: echo "::set-env name=CARGO_HOME::$(pwd)/.cargo_home" - name: Cache - uses: denoland/github-actions-cache@stable-prerelease + uses: actions/cache@v2 with: # Note: crates from the denoland/deno git repo always get rebuilt, # and their outputs ('deno', 'libdeno.rlib' etc.) are quite big, diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000000..39e827a8492480 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,4 @@ +Deno uses [Rust's Code of +Conduct](https://www.rust-lang.org/policies/code-of-conduct). In the +forums, every community member must follow the rules and values expressed +there. Please email ry@tinyclouds.org to report any instance of misconduct. diff --git a/Cargo.lock b/Cargo.lock index 8d7391307cb888..9eb1e61228ac66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,6 +10,15 @@ dependencies = [ "regex", ] +[[package]] +name = "addr2line" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a49806b9dadc843c61e7c97e72490ad7f7220ae249012fbda9ad0609457c0543" +dependencies = [ + "gimli", +] + [[package]] name = "adler32" version = "1.0.4" @@ -18,9 +27,9 @@ checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" [[package]] name = "ahash" -version = "0.3.2" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0989268a37e128d4d7a8028f1c60099430113fdbc70419010601ce51a228e4fe" +checksum = "2f3e0bf23f51883cce372d5d5892211236856e4bb37fb942e1eb135ee0f146e3" dependencies = [ "const-random", ] @@ -66,9 +75,9 @@ checksum = "33954243bd79057c2de7338850b85983a44588021f8a5fee574a8888c6de4344" [[package]] name = "arc-swap" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825" +checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62" [[package]] name = "arrayref" @@ -90,17 +99,17 @@ checksum = "7d96b5937e2a8b8dd9eac561c192f7fef2ab0cbc06c445e67b9e637ab158c52b" dependencies = [ "darling", "pmutil", - "proc-macro2 1.0.10", - "quote 1.0.3", + "proc-macro2 1.0.18", + "quote 1.0.6", "swc_macros_common", - "syn 1.0.17", + "syn 1.0.30", ] [[package]] name = "async-compression" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3e94dc198f93aa4107649e68fbb7f4d759398643d36f498eb72a2ee8256652" +checksum = "ae84766bab9f774e32979583ba56d6af8c701288c6dc99144819d5d2ee0b170f" dependencies = [ "brotli", "bytes 0.5.4", @@ -135,26 +144,17 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.46" +version = "0.3.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e692897359247cc6bb902933361652380af0f1b7651ae5c5013407f30e109e" +checksum = "0df2f85c8a2abbe3b7d7e748052fdd9b76a0458fdeb16ad4223f5eca78c7c130" dependencies = [ - "backtrace-sys", + "addr2line", "cfg-if", "libc", + "object", "rustc-demangle", ] -[[package]] -name = "backtrace-sys" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "base64" version = "0.10.1" @@ -172,9 +172,9 @@ checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" [[package]] name = "base64" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5ca2cd0adc3f48f9e9ea5a6bbdf9ccc0bfade884847e484d452414c7ccffb3" +checksum = "53d1ccbaf7d9ec9537465a97bf19edc1a4e158ecb49fc16178202238c569cc42" [[package]] name = "bitflags" @@ -227,9 +227,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a9f2b517b96b19d8f91c1ff5b1cf498e688850b32eae5d58e02d15c4d4fdc0c" +checksum = "1052e1c3b8d4d80eb84a8b94f0a1498797b5fb96314c001156a1c761940ef4ec" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.2.1" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" [[package]] name = "byte-tools" @@ -287,9 +287,9 @@ checksum = "5ba7d7f7b201dfcbc314b14f2176c92f8ba521dab538b40e426ffed25ed7cd80" [[package]] name = "cc" -version = "1.0.52" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d87b23d6a92cd03af510a5ade527033f6aa6fa92161e2d5863a907d4c5e31d" +checksum = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311" [[package]] name = "cfg-if" @@ -309,9 +309,9 @@ dependencies = [ [[package]] name = "clap" -version = "2.33.0" +version = "2.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" dependencies = [ "ansi_term", "atty", @@ -357,22 +357,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" -[[package]] -name = "core-foundation" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" - [[package]] name = "crc32fast" version = "1.2.0" @@ -403,15 +387,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "ct-logs" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113" -dependencies = [ - "sct", -] - [[package]] name = "darling" version = "0.10.2" @@ -430,10 +405,10 @@ checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.10", - "quote 1.0.3", + "proc-macro2 1.0.18", + "quote 1.0.6", "strsim 0.9.3", - "syn 1.0.17", + "syn 1.0.30", ] [[package]] @@ -443,8 +418,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ "darling_core", - "quote 1.0.3", - "syn 1.0.17", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -460,19 +435,21 @@ dependencies = [ [[package]] name = "deno" -version = "1.0.2" +version = "1.0.5" dependencies = [ "atty", - "base64 0.12.0", + "base64 0.12.1", "byteorder", "bytes 0.5.4", "clap", "deno_core", + "deno_lint", "deno_typescript", "dirs", + "dissimilar", "dlopen", "dprint-plugin-typescript", - "futures 0.3.4", + "futures 0.3.5", "fwdansi", "glob", "http", @@ -507,17 +484,17 @@ dependencies = [ "walkdir", "warp", "webpki", - "webpki-roots 0.19.0", + "webpki-roots", "winapi 0.3.8", ] [[package]] name = "deno_core" -version = "0.45.2" +version = "0.47.1" dependencies = [ "derive_deref", "downcast-rs", - "futures 0.3.4", + "futures 0.3.5", "lazy_static", "libc", "log 0.4.8", @@ -527,9 +504,24 @@ dependencies = [ "url 2.1.1", ] +[[package]] +name = "deno_lint" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a3ec96c92609aa121d085f3a1351d3836e55b78a4b8ce79ea771c2ad9bd80b" +dependencies = [ + "lazy_static", + "regex", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "swc_ecma_visit", +] + [[package]] name = "deno_typescript" -version = "0.45.2" +version = "0.47.1" dependencies = [ "deno_core", "serde", @@ -578,6 +570,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "dissimilar" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39de161cd2ebbd6e5783db53a82a47b6a47dcfef754130839603561745528b94" + [[package]] name = "dlopen" version = "0.1.8" @@ -609,18 +607,18 @@ checksum = "52ba6eb47c2131e784a38b726eb54c1e1484904f013e576a25354d0124161af6" [[package]] name = "dprint-core" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b62634626adceb7abdab6e4dd5fc8cedcb177b2cb80a6491519092370afa037" +checksum = "e932279bfbf2e280772f1708be5502d7ff2de6a3afb388d24a38f4fdf9fe91f5" dependencies = [ "serde", ] [[package]] name = "dprint-plugin-typescript" -version = "0.18.5" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4b628385a13a592afdd6391167921f62f1c1adc1f7b098ecf7e930ab5cc702" +checksum = "50b1a94e7824e2b41a4c7eddd39516070c283bc7f525a76fdc9230efbf2b56b5" dependencies = [ "dprint-core", "serde", @@ -644,9 +642,9 @@ checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" [[package]] name = "encoding_rs" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d03faa7fe0c1431609dfad7bbe827af30f82e1e2ae6f7ee4fca6bd764bc28" +checksum = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" dependencies = [ "cfg-if", ] @@ -658,9 +656,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e57153e35187d51f08471d5840459ff29093473e7bedd004a1414985aab92f3" dependencies = [ "pmutil", - "proc-macro2 1.0.10", + "proc-macro2 1.0.18", "swc_macros_common", - "syn 1.0.17", + "syn 1.0.30", ] [[package]] @@ -676,9 +674,9 @@ dependencies = [ [[package]] name = "failure" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" dependencies = [ "backtrace", ] @@ -691,9 +689,9 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "filetime" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f59efc38004c988e4201d11d263b8171f49a2e7ec0bdbb71773433f271504a5e" +checksum = "affc17579b132fc2461adf7c575cc6e8b134ebca52c51f5411388965227dc695" dependencies = [ "cfg-if", "libc", @@ -715,9 +713,9 @@ dependencies = [ [[package]] name = "fnv" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "from_variant" @@ -726,9 +724,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "039885ad6579a86b94ad8df696cce8c530da496bf7b07b12fec8d6c4cd654bb9" dependencies = [ "pmutil", - "proc-macro2 1.0.10", + "proc-macro2 1.0.18", "swc_macros_common", - "syn 1.0.17", + "syn 1.0.30", ] [[package]] @@ -780,9 +778,9 @@ checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" [[package]] name = "futures" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c329ae8753502fb44ae4fc2b622fa2a94652c41e795143765ba0927f92ab780" +checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613" dependencies = [ "futures-channel", "futures-core", @@ -795,9 +793,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" dependencies = [ "futures-core", "futures-sink", @@ -805,15 +803,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" +checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" [[package]] name = "futures-executor" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f674f3e1bcb15b37284a90cedf55afdba482ab061c407a9c0ebbd0f3109741ba" +checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314" dependencies = [ "futures-core", "futures-task", @@ -823,39 +821,42 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" +checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" [[package]] name = "futures-macro" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" +checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" dependencies = [ "proc-macro-hack", - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] name = "futures-sink" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" +checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" [[package]] name = "futures-task" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" +checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" +dependencies = [ + "once_cell", +] [[package]] name = "futures-util" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" +checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" dependencies = [ "futures 0.1.29", "futures-channel", @@ -865,6 +866,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", + "pin-project", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -911,6 +913,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c" + [[package]] name = "glob" version = "0.3.0" @@ -919,9 +927,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "377038bf3c89d18d6ca1431e7a5027194fbd724ca10592b9487ede5e8e144f42" +checksum = "79b7246d7e4b979c03fa093da39cfb3617a96bbeee6310af63991668d7e843ff" dependencies = [ "bytes 0.5.4", "fnv", @@ -942,7 +950,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed18eb2459bf1a09ad2d6b1547840c3e5e62882fa09b9a6a20b1de8e3228848f" dependencies = [ - "base64 0.12.0", + "base64 0.12.1", "bitflags", "bytes 0.5.4", "headers-core", @@ -963,9 +971,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.10" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e" +checksum = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71" dependencies = [ "libc", ] @@ -999,9 +1007,9 @@ checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" [[package]] name = "hyper" -version = "0.13.5" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14" +checksum = "a6e7655b9594024ad0ee439f3b5a7299369dc2a3f459b47c696f9ff676f9aa1f" dependencies = [ "bytes 0.5.4", "futures-channel", @@ -1013,8 +1021,8 @@ dependencies = [ "httparse", "itoa", "log 0.4.8", - "net2", "pin-project", + "socket2", "time", "tokio", "tower-service", @@ -1028,12 +1036,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac965ea399ec3a25ac7d13b8affd4b8f39325cca00858ddf5eb29b79e6b14b08" dependencies = [ "bytes 0.5.4", - "ct-logs", "futures-util", "hyper", "log 0.4.8", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "webpki", @@ -1083,9 +1089,9 @@ checksum = "4bac95d9aa0624e7b78187d6fb8ab012b41d9f6f54b1bcb61e61c4845f8357ec" [[package]] name = "indexmap" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" +checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" dependencies = [ "autocfg 1.0.0", ] @@ -1136,9 +1142,9 @@ checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" [[package]] name = "js-sys" -version = "0.3.37" +version = "0.3.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a27d435371a2fa5b6d2b028a74bbdb1234f308da363226a2854ca3ff8ba7055" +checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" dependencies = [ "wasm-bindgen", ] @@ -1167,9 +1173,9 @@ checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" [[package]] name = "libc" -version = "0.2.69" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" +checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" [[package]] name = "lock_api" @@ -1265,9 +1271,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" dependencies = [ "cfg-if", "fuchsia-zircon", @@ -1302,15 +1308,15 @@ checksum = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" dependencies = [ "log 0.4.8", "mio", - "miow 0.3.3", + "miow 0.3.4", "winapi 0.3.8", ] [[package]] name = "mio-uds" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" dependencies = [ "iovec", "libc", @@ -1331,9 +1337,9 @@ dependencies = [ [[package]] name = "miow" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "396aa0f2003d7df8395cb93e09871561ccc3e785f0acb369170e8cc74ddf9226" +checksum = "22dfdd1d51b2639a5abd17ed07005c3af05fb7a2a3b1a1d0d7af1000a520c1c7" dependencies = [ "socket2", "winapi 0.3.8", @@ -1359,9 +1365,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" +checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" dependencies = [ "cfg-if", "libc", @@ -1441,19 +1447,25 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" dependencies = [ "hermit-abi", "libc", ] +[[package]] +name = "object" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cbca9424c482ee628fa549d9c812e2cd22f1180b9222c9200fdfa6eb31aecb2" + [[package]] name = "once_cell" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" +checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" [[package]] name = "opaque-debug" @@ -1461,17 +1473,11 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" -[[package]] -name = "openssl-probe" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" - [[package]] name = "os_pipe" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d06355a7090ce852965b2d08e11426c315438462638c6d721448d0b47aa22" +checksum = "fb233f06c2307e1f5ce2ecad9f8121cffbbee2c95428f44ea85222e460d0d213" dependencies = [ "libc", "winapi 0.3.8", @@ -1607,40 +1613,40 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" dependencies = [ - "siphasher 0.3.2", + "siphasher 0.3.3", ] [[package]] name = "pin-project" -version = "0.4.8" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7804a463a8d9572f13453c516a5faea534a2403d7ced2f0c7e100eeff072772c" +checksum = "edc93aeee735e60ecb40cf740eb319ff23eab1c5748abfdb5c180e4ce49f7791" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.8" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f" +checksum = "e58db2081ba5b4c93bd6be09c40fd36cb9193a8336c384f3b40012e531aa7e40" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] name = "pin-project-lite" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" +checksum = "9df32da11d84f3a7d70205549562966279adb900e080fad3dccd8e64afccf0ad" [[package]] name = "pin-utils" -version = "0.1.0-alpha.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pmutil" @@ -1648,16 +1654,16 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] name = "ppv-lite86" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" +checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" [[package]] name = "precomputed-hash" @@ -1667,9 +1673,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "proc-macro-hack" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" +checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" [[package]] name = "proc-macro-nested" @@ -1688,9 +1694,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.10" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" dependencies = [ "unicode-xid 0.2.0", ] @@ -1722,11 +1728,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.3" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" +checksum = "54a21852a652ad6f610c9510194f398ff6f8692e334fd1145fed931f7fbe44ea" dependencies = [ - "proc-macro2 1.0.10", + "proc-macro2 1.0.18", ] [[package]] @@ -1927,9 +1933,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.7" +version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" dependencies = [ "aho-corasick", "memchr", @@ -1939,9 +1945,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" [[package]] name = "remove_dir_all" @@ -1954,12 +1960,12 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.4" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b81e49ddec5109a9dcfc5f2a317ff53377c915e9ae9d4f2fb50914b85614e2" +checksum = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" dependencies = [ "async-compression", - "base64 0.11.0", + "base64 0.12.1", "bytes 0.5.4", "encoding_rs", "futures-core", @@ -1978,22 +1984,21 @@ dependencies = [ "rustls", "serde", "serde_urlencoded", - "time", "tokio", "tokio-rustls", "url 2.1.1", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.18.0", + "webpki-roots", "winreg", ] [[package]] name = "ring" -version = "0.16.13" +version = "0.16.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703516ae74571f24b465b4a1431e81e2ad51336cb0ded733a55a1aa3eccac196" +checksum = "06b3fefa4f12272808f809a0af618501fdaba41a58963c5fb72238ab0be09603" dependencies = [ "cc", "libc", @@ -2044,23 +2049,11 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustls-native-certs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75ffeb84a6bd9d014713119542ce415db3a3e4748f0bfce1e1416cd224a23a5" -dependencies = [ - "openssl-probe", - "rustls", - "schannel", - "security-framework", -] - [[package]] name = "rusty_v8" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acb0ad56a57c42009a8d16df5fa94ae882ad0ffe0e88fe1a23b261b3affbccf2" +checksum = "66491597ce62f02c48f0194fc9574ec0b50c648e84947400ba13882394b6d56c" dependencies = [ "bitflags", "cargo_gn", @@ -2090,9 +2083,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "safemem" @@ -2109,16 +2102,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "039c25b130bd8c1321ee2d7de7fde2659fa9c2744e4bb29711cfc852ea53cd19" -dependencies = [ - "lazy_static", - "winapi 0.3.8", -] - [[package]] name = "scoped-tls" version = "1.0.0" @@ -2147,29 +2130,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "security-framework" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572dfa3a0785509e7a44b5b4bebcf94d41ba34e9ed9eb9df722545c3b3c4144a" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb15a5fec93b7021b8a9e96009c5d8d51c15673569f7c0f6b7204e5b7b404f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "semver" version = "0.9.0" @@ -2193,29 +2153,29 @@ checksum = "b46e1121e8180c12ff69a742aabc4f310542b6ccb69f1691689ac17fdf8618aa" [[package]] name = "serde" -version = "1.0.106" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" +checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.106" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" +checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] name = "serde_json" -version = "1.0.52" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7894c8ed05b7a3a279aeb79025fdec1d3158080b75b98a08faf2806bb799edd" +checksum = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2" dependencies = [ "indexmap", "itoa", @@ -2265,9 +2225,9 @@ checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" [[package]] name = "siphasher" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e88f89a550c01e4cd809f3df4f52dc9e939f3273a2017eabd5c6d12fd98bb23" +checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" [[package]] name = "slab" @@ -2286,9 +2246,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" +checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" [[package]] name = "socket2" @@ -2351,8 +2311,8 @@ checksum = "f24c8e5e19d22a726626f1a5e16fe15b132dcf21d10177fa5a45ce7962996b97" dependencies = [ "phf_generator 0.8.0", "phf_shared 0.8.0", - "proc-macro2 1.0.10", - "quote 1.0.3", + "proc-macro2 1.0.18", + "quote 1.0.6", ] [[package]] @@ -2362,10 +2322,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94fdb6536756cfd35ee18b9a9972ab2a699d405cc57e0ad0532022960f30d581" dependencies = [ "pmutil", - "proc-macro2 1.0.10", - "quote 1.0.3", + "proc-macro2 1.0.18", + "quote 1.0.6", "swc_macros_common", - "syn 1.0.17", + "syn 1.0.30", ] [[package]] @@ -2429,9 +2389,9 @@ dependencies = [ [[package]] name = "swc_ecma_parser" -version = "0.23.0" +version = "0.24.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ddb997ff7671faf7e55040233ed814bed1f69b6d1ad7a0fe78948ebf42d3d6a" +checksum = "3b06bb3791cf307722bb7645f36509217649d36ac21e82493a37a8adcdce1d83" dependencies = [ "either", "enum_kind", @@ -2440,7 +2400,7 @@ dependencies = [ "once_cell", "regex", "serde", - "smallvec 1.3.0", + "smallvec 1.4.0", "swc_atoms", "swc_common", "swc_ecma_ast", @@ -2455,17 +2415,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8798810e2c79b884cf238bcb72b4bd12375121ee91724f1ceeb54b6e38a138e7" dependencies = [ "pmutil", - "proc-macro2 1.0.10", - "quote 1.0.3", + "proc-macro2 1.0.18", + "quote 1.0.6", "swc_macros_common", - "syn 1.0.17", + "syn 1.0.30", ] [[package]] name = "swc_ecma_visit" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "802b13ac5811ebc6897c9a1ed999aa9ca3dd6faf8e664b9df44e162e2b3a03ed" +checksum = "43d08204c4454eee8dcef3f5f4b9f2104abd1540740e93ebc49677777553b7b5" dependencies = [ "num-bigint", "swc_atoms", @@ -2476,16 +2436,16 @@ dependencies = [ [[package]] name = "swc_ecma_visit_macros" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9083af117bcb987631d99129ce57e634315267d84521fd562926135fa49bee38" +checksum = "1a32877a341800d772d95858c24344ce00f68361763eae5fa97187e6be136c4b" dependencies = [ "Inflector", "pmutil", - "proc-macro2 1.0.10", - "quote 1.0.3", + "proc-macro2 1.0.18", + "quote 1.0.6", "swc_macros_common", - "syn 1.0.17", + "syn 1.0.30", ] [[package]] @@ -2495,9 +2455,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18a9f27d290938370597d363df9a77ba4be8e2bc99f32f69eb5245cdeed3c512" dependencies = [ "pmutil", - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] @@ -2513,20 +2473,20 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.17" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" +checksum = "93a56fabc59dce20fe48b6c832cc249c713e7ed88fa28b0ee0a3bfcaae5fe4e2" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", + "proc-macro2 1.0.18", + "quote 1.0.6", "unicode-xid 0.2.0", ] [[package]] name = "sys-info" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0fb3fecc8cf6ffd2b75b19f0b1e56770fc8d370fed8bb82ce90e29014951c3" +checksum = "e5cfbd84f86389198ade41b439f72a5b1b3a8ba728e61cd589e1720d0df44c39" dependencies = [ "cc", "libc", @@ -2560,7 +2520,7 @@ name = "test_plugin" version = "0.0.1" dependencies = [ "deno_core", - "futures 0.3.4", + "futures 0.3.5", ] [[package]] @@ -2583,20 +2543,19 @@ dependencies = [ [[package]] name = "time" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "redox_syscall", "winapi 0.3.8", ] [[package]] name = "tokio" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c1d570eb1a36f0345a5ce9c6c6e665b70b73d11236912c0b477616aeec47b1" +checksum = "d099fa27b9702bed751524694adbe393e18b36b204da91eb1cbbbbb4a5ee2d58" dependencies = [ "bytes 0.5.4", "fnv", @@ -2633,16 +2592,16 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", ] [[package]] name = "tokio-rustls" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4adb8b3e5f86b707f1b54e7c15b6de52617a823608ccda98a15d3a24222f265a" +checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4" dependencies = [ "futures-core", "rustls", @@ -2656,7 +2615,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8b8fe88007ebc363512449868d7da4389c9400072a3f666f212c7280082882a" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "log 0.4.8", "pin-project", "tokio", @@ -2738,7 +2697,7 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.9.1", + "version_check 0.9.2", ] [[package]] @@ -2756,7 +2715,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" dependencies = [ - "smallvec 1.3.0", + "smallvec 1.4.0", ] [[package]] @@ -2831,9 +2790,9 @@ checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" [[package]] name = "utime" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "055058552ca15c566082fc61da433ae678f78986a6f16957e33162d1b218792a" +checksum = "cfab4578d925146644058fa81870b9dafd132365259758fb9e6e76b89a303494" dependencies = [ "kernel32-sys", "libc", @@ -2851,9 +2810,9 @@ dependencies = [ [[package]] name = "vec_map" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" @@ -2863,9 +2822,9 @@ checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" [[package]] name = "version_check" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" [[package]] name = "void" @@ -2896,12 +2855,12 @@ dependencies = [ [[package]] name = "warp" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54cd1e2b3eb3539284d88b76a9afcf5e20f2ef2fab74db5b21a1c30d7d945e82" +checksum = "0e95175b7a927258ecbb816bdada3cc469cb68593e7940b96a60f4af366a9970" dependencies = [ "bytes 0.5.4", - "futures 0.3.4", + "futures 0.3.5", "headers", "http", "hyper", @@ -2928,9 +2887,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.60" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc57ce05287f8376e998cbddfb4c8cb43b84a7ec55cf4551d7c00eef317a47f" +checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0" dependencies = [ "cfg-if", "serde", @@ -2940,24 +2899,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.60" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967d37bf6c16cca2973ca3af071d0a2523392e4a594548155d89a678f4237cd" +checksum = "ded84f06e0ed21499f6184df0e0cb3494727b0c5da89534e0fcc55c51d812101" dependencies = [ "bumpalo", "lazy_static", "log 0.4.8", - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add542ea1ac7fdaa9dc25e031a6af33b7d63376292bd24140c637d00d1c312a" +checksum = "64487204d863f109eb77e8462189d111f27cb5712cc9fdb3461297a76963a2f6" dependencies = [ "cfg-if", "js-sys", @@ -2967,38 +2926,38 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.60" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bd151b63e1ea881bb742cd20e1d6127cef28399558f3b5d415289bc41eee3a4" +checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3" dependencies = [ - "quote 1.0.3", + "quote 1.0.6", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.60" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68a5b36eef1be7868f668632863292e37739656a80fc4b9acec7b0bd35a4931" +checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92" dependencies = [ - "proc-macro2 1.0.10", - "quote 1.0.3", - "syn 1.0.17", + "proc-macro2 1.0.18", + "quote 1.0.6", + "syn 1.0.30", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.60" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf76fe7d25ac79748a37538b7daeed1c7a6867c92d3245c12c6222e4a20d639" +checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd" [[package]] name = "web-sys" -version = "0.3.37" +version = "0.3.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6f51648d8c56c366144378a33290049eafdd784071077f6fe37dae64c1c4cb" +checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17" dependencies = [ "js-sys", "wasm-bindgen", @@ -3006,23 +2965,14 @@ dependencies = [ [[package]] name = "webpki" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f50e1972865d6b1adb54167d1c8ed48606004c2c9d0ea5f1eeb34d95e863ef" +checksum = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae" dependencies = [ "ring", "untrusted", ] -[[package]] -name = "webpki-roots" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cd5736df7f12a964a5067a12c62fa38e1bd8080aff1f80bc29be7c80d19ab4" -dependencies = [ - "webpki", -] - [[package]] name = "webpki-roots" version = "0.19.0" @@ -3072,9 +3022,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi 0.3.8", ] @@ -3087,9 +3037,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winreg" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ "winapi 0.3.8", ] diff --git a/Releases.md b/Releases.md index 466790ca7d26d1..8d86ce47df351b 100644 --- a/Releases.md +++ b/Releases.md @@ -6,6 +6,95 @@ https://github.com/denoland/deno/releases We also have one-line install commands at https://github.com/denoland/deno_install +### 1.0.5 / 2020.06.03 + +Changes in the CLI: + +- fix(fetch): Support 101 status code (#6059) +- fix: REPL BorrowMutError panic (#6055) +- fix: dynamic import BorrowMutError (#6065) +- upgrade: dprint 0.19.1 and swc_ecma_parser 0.24.3 (#6068) +- upgrade: rusty_v8 0.5.0 (#6070) + +Changes in std version 0.56.0: + +- feat(std/testing): benching progress callback (#5941) +- feat(std/encoding): add base64url module (#5976) +- fix(std/testing/asserts): Format values in assertArrayContains() (#6060) + +### 1.0.4 / 2020.06.02 + +Changes in the CLI: + +- feat(core): Ops can take several zero copy buffers (#4788) +- fix(bundle): better size output (#5997) +- fix(cli): Deno.remove() fails to remove unix socket (#5967) +- fix(cli): compile TS dependencies of JS files (#6000) +- fix(cli): ES private fields parsing in SWC (#5964) +- fix(cli): Better use of @ts-expect-error (#6038) +- fix(cli): media type for .cjs and application/node (#6005) +- fix(doc): remove JSDoc comment truncation (#6031) +- fix(cli/js/web): Body.bodyUsed should use IsReadableStreamDisturbed +- fix(cli/js/web): formData parser for binary files in fetch() (#6015) +- fix(cli/js/web): set null body for null-body status in fetch() (#5980) +- fix(cli/js/web): network error on multiple redirects in fetch() (#5985) +- fix(cli/js/web): Headers.name and FormData.name (#5994) +- upgrade: Rust crates (#5959, #6032) + +Changes in std version 0.55.0: + +- feat(std/hash): add Sha512 and HmacSha512 (#6009) +- feat(std/http) support code 103 Early Hints (#6021) +- feat(std/http): add TooEarly status code (#5999) +- feat(std/io): add LimitedReader (#6026) +- feat(std/log): buffered file logging (#6014) +- feat(std/mime/multipart): Added multiple FormFile input (#6027) +- feat(std/node): add util.type.isDate (#6029) +- fix(std/http): file server not closing files (#5952) +- fix(std/path): support browsers (#6003) + +### 1.0.3 / 2020.05.29 + +Changes in the CLI: + +- fix: Add unstable checks for Deno.dir and Diagnostics (#5750) +- fix: Add unstable checks for unix transport (#5818) +- fix: Create HTTP cache lazily (#5795) +- fix: Dependency analysis in TS compiler (#5817, #5785, #5870) +- fix: Expose Error.captureStackTrace (#5254) +- fix: Improved typechecking error for unstable props (#5503) +- fix: REPL evaluates in strict mode (#5565) +- fix: Write lock file before running any code (#5794) +- fix(debugger): BorrowMutError when evaluating expression in inspector console + (#5822) +- fix(doc): Handle comments at the top of the file (#5891) +- fix(fmt): Handle formatting UTF-8 w/ BOM files (#5881) +- fix(permissions): Fix CWD and exec path leaks (#5642) +- fix(web/blob): DenoBlob name (#5879) +- fix(web/console): Hide `values` for console.table if display not necessary + (#5914) +- fix(web/console): Improve indentation when displaying objects with console.log + (#5909) +- fix(web/encoding): atob should throw dom exception (#5730) +- fix(web/fetch): Make Response constructor standard (#5787) +- fix(web/fetch): Allow ArrayBuffer as Fetch request body (#5831) +- fix(web/formData): Set default filename for Blob to (#5907) +- upgrade: dprint to 0.19.0 (#5899) + +Changes in std version 0.54.0: + +- feat(std/encoding): Add base64 (#5811) +- feat(std/http): Handle .wasm files in file_server (#5896) +- feat(std/node): Add link/linkSync polyfill (#5930) +- feat(std/node): fs.writeFile/sync path can now be an URL (#5652) +- feat(std/testing): Return results in benchmark promise (#5842) +- fix(std/http): readTrailer evaluates header names by case-insensitive (#4902) +- fix(std/log): Improve the calculation of byte length (#5819) +- fix(std/log): Fix FileHandler test with mode 'x' on non-English systems + (#5757) +- fix(std/log): Use writeAllSync instead of writeSync (#5868) +- fix(std/testing/asserts): Support browsers (#5847) + ### 1.0.2 / 2020.05.22 Changes in the CLI: diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 36e58b0e086c5c..01a240d3a6fb7f 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno" -version = "1.0.2" +version = "1.0.5" license = "MIT" authors = ["the Deno authors"] edition = "2018" @@ -15,52 +15,54 @@ name = "deno" path = "main.rs" [build-dependencies] -deno_core = { path = "../core", version = "0.45.0" } -deno_typescript = { path = "../deno_typescript", version = "0.45.0" } +deno_core = { path = "../core", version = "0.47.1" } +deno_typescript = { path = "../deno_typescript", version = "0.47.1" } [dependencies] -deno_core = { path = "../core", version = "0.45.0" } -deno_typescript = { path = "../deno_typescript", version = "0.45.0" } +deno_core = { path = "../core", version = "0.47.1" } +deno_lint = { version = "0.1.7" } +deno_typescript = { path = "../deno_typescript", version = "0.47.1" } atty = "0.2.14" -base64 = "0.12.0" +base64 = "0.12.1" bytes = "0.5.4" byteorder = "1.3.4" -clap = "2.33.0" +clap = "2.33.1" dirs = "2.0.2" +dissimilar = "1.0" dlopen = "0.1.8" -dprint-plugin-typescript = "0.18.5" -futures = { version = "0.3.4", features = ["compat", "io-compat"] } +dprint-plugin-typescript = "0.19.2" +futures = { version = "0.3.5", features = ["compat", "io-compat"] } glob = "0.3.0" http = "0.2.1" indexmap = "1.3.2" lazy_static = "1.4.0" -libc = "0.2.69" +libc = "0.2.71" log = "0.4.8" notify = "5.0.0-pre.2" rand = "0.7.3" -regex = "1.3.7" -reqwest = { version = "0.10.4", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli"] } -ring = "0.16.13" +regex = "1.3.9" +reqwest = { version = "0.10.6", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli"] } +ring = "0.16.14" rustyline = "6.1.2" -serde = { version = "1.0.106", features = ["derive"] } -serde_derive = "1.0.106" -serde_json = { version = "1.0.52", features = [ "preserve_order" ] } -sys-info = "0.6.1" +serde = { version = "1.0.111", features = ["derive"] } +serde_derive = "1.0.111" +serde_json = { version = "1.0.53", features = [ "preserve_order" ] } +sys-info = "0.7.0" sourcemap = "5.0.0" tempfile = "3.1.0" termcolor = "1.1.0" -tokio = { version = "0.2.20", features = ["rt-core", "tcp", "udp", "uds", "process", "fs", "blocking", "sync", "io-std", "macros", "time"] } -tokio-rustls = "0.13.0" +tokio = { version = "0.2.21", features = ["rt-core", "tcp", "udp", "uds", "process", "fs", "blocking", "sync", "io-std", "macros", "time"] } +tokio-rustls = "0.13.1" url = "2.1.1" -utime = "0.2.1" -webpki = "0.21.2" +utime = "0.3.0" +webpki = "0.21.3" webpki-roots = "0.19.0" walkdir = "2.3.1" -warp = "0.2.2" +warp = "0.2.3" semver-parser = "0.9.0" uuid = { version = "0.8.1", features = ["v4"] } -swc_ecma_visit = "0.4.0" +swc_ecma_visit = "0.5.1" [target.'cfg(windows)'.dependencies] winapi = "0.3.8" @@ -70,7 +72,7 @@ fwdansi = "1.1.0" nix = "0.17.0" [dev-dependencies] -os_pipe = "0.9.1" +os_pipe = "0.9.2" # Used for testing inspector. Keep in-sync with warp. tokio-tungstenite = { version = "0.10.1", features = ["connect"] } diff --git a/cli/colors.rs b/cli/colors.rs index 764704a5b8d1ee..b9a5a7353e5a52 100644 --- a/cli/colors.rs +++ b/cli/colors.rs @@ -52,6 +52,12 @@ pub fn red_bold(s: String) -> impl fmt::Display { style(&s, style_spec) } +pub fn green_bold(s: String) -> impl fmt::Display { + let mut style_spec = ColorSpec::new(); + style_spec.set_fg(Some(Ansi256(10))).set_bold(true); + style(&s, style_spec) +} + pub fn italic_bold(s: String) -> impl fmt::Display { let mut style_spec = ColorSpec::new(); style_spec.set_bold(true).set_italic(true); @@ -64,6 +70,18 @@ pub fn black_on_white(s: String) -> impl fmt::Display { style(&s, style_spec) } +pub fn white_on_red(s: String) -> impl fmt::Display { + let mut style_spec = ColorSpec::new(); + style_spec.set_bg(Some(Red)).set_fg(Some(White)); + style(&s, style_spec) +} + +pub fn white_on_green(s: String) -> impl fmt::Display { + let mut style_spec = ColorSpec::new(); + style_spec.set_bg(Some(Ansi256(10))).set_fg(Some(White)); + style(&s, style_spec) +} + pub fn yellow(s: String) -> impl fmt::Display { let mut style_spec = ColorSpec::new(); style_spec.set_fg(Some(Ansi256(11))); diff --git a/cli/deno_dir.rs b/cli/deno_dir.rs index 3224fbf4674978..1447682ec63eac 100644 --- a/cli/deno_dir.rs +++ b/cli/deno_dir.rs @@ -40,7 +40,7 @@ impl DenoDir { root, gen_cache: DiskCache::new(&gen_path), }; - deno_dir.gen_cache.ensure_location()?; + deno_dir.gen_cache.ensure_dir_exists(&gen_path)?; Ok(deno_dir) } diff --git a/cli/diff.rs b/cli/diff.rs new file mode 100644 index 00000000000000..e6d7fa4bb0ca48 --- /dev/null +++ b/cli/diff.rs @@ -0,0 +1,171 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::colors; +use dissimilar::{diff as difference, Chunk}; +use std::fmt; +use std::fmt::Write; + +fn fmt_add() -> String { + format!("{}", colors::green_bold("+".to_string())) +} + +fn fmt_add_text(x: String) -> String { + format!("{}", colors::green(x)) +} + +fn fmt_add_text_highlight(x: String) -> String { + format!("{}", colors::white_on_green(x)) +} + +fn fmt_rem() -> String { + format!("{}", colors::red_bold("-".to_string())) +} + +fn fmt_rem_text(x: String) -> String { + format!("{}", colors::red(x)) +} + +fn fmt_rem_text_highlight(x: String) -> String { + format!("{}", colors::white_on_red(x)) +} + +fn write_line_diff( + diff: &mut String, + orig_line: &mut usize, + edit_line: &mut usize, + line_number_width: usize, + orig: &mut String, + edit: &mut String, +) -> fmt::Result { + let split = orig.split('\n').enumerate(); + for (i, s) in split { + write!( + diff, + "{:0width$}{} ", + *orig_line + i, + colors::gray("|".to_string()), + width = line_number_width + )?; + write!(diff, "{}", fmt_rem())?; + write!(diff, "{}", s)?; + writeln!(diff)?; + } + + let split = edit.split('\n').enumerate(); + for (i, s) in split { + write!( + diff, + "{:0width$}{} ", + *edit_line + i, + colors::gray("|".to_string()), + width = line_number_width + )?; + write!(diff, "{}", fmt_add())?; + write!(diff, "{}", s)?; + writeln!(diff)?; + } + + *orig_line += orig.split('\n').count(); + *edit_line += edit.split('\n').count(); + + orig.clear(); + edit.clear(); + + Ok(()) +} + +/// Print diff of the same file_path, before and after formatting. +/// +/// Diff format is loosely based on Github diff formatting. +pub fn diff(orig_text: &str, edit_text: &str) -> Result { + let lines = edit_text.split('\n').count(); + let line_number_width = lines.to_string().chars().count(); + + let mut diff = String::new(); + + let mut text1 = orig_text.to_string(); + let mut text2 = edit_text.to_string(); + + if !text1.ends_with('\n') { + writeln!(text1)?; + } + if !text2.ends_with('\n') { + writeln!(text2)?; + } + + let mut orig_line: usize = 1; + let mut edit_line: usize = 1; + let mut orig: String = String::new(); + let mut edit: String = String::new(); + let mut changes = false; + + let chunks = difference(&text1, &text2); + for chunk in chunks { + match chunk { + Chunk::Delete(s) => { + let split = s.split('\n').enumerate(); + for (i, s) in split { + if i > 0 { + orig.push_str("\n"); + } + orig.push_str(&fmt_rem_text_highlight(s.to_string())); + } + changes = true + } + Chunk::Insert(s) => { + let split = s.split('\n').enumerate(); + for (i, s) in split { + if i > 0 { + edit.push_str("\n"); + } + edit.push_str(&fmt_add_text_highlight(s.to_string())); + } + changes = true + } + Chunk::Equal(s) => { + let split = s.split('\n').enumerate(); + for (i, s) in split { + if i > 0 { + if changes { + write_line_diff( + &mut diff, + &mut orig_line, + &mut edit_line, + line_number_width, + &mut orig, + &mut edit, + )?; + changes = false + } else { + orig.clear(); + edit.clear(); + orig_line += 1; + edit_line += 1; + } + } + orig.push_str(&fmt_rem_text(s.to_string())); + edit.push_str(&fmt_add_text(s.to_string())); + } + } + } + } + Ok(diff) +} + +#[test] +fn test_diff() { + let simple_console_log_unfmt = "console.log('Hello World')"; + let simple_console_log_fmt = "console.log(\"Hello World\");"; + assert_eq!( + colors::strip_ansi_codes( + &diff(simple_console_log_unfmt, simple_console_log_fmt).unwrap() + ), + "1| -console.log('Hello World')\n1| +console.log(\"Hello World\");\n" + ); + + let line_number_unfmt = "\n\n\n\nconsole.log(\n'Hello World'\n)"; + let line_number_fmt = "console.log(\n\"Hello World\"\n);"; + assert_eq!( + colors::strip_ansi_codes(&diff(line_number_unfmt, line_number_fmt).unwrap()), + "1| -\n2| -\n3| -\n4| -\n5| -console.log(\n1| +console.log(\n6| -'Hello World'\n2| +\"Hello World\"\n7| -)\n3| +);\n" + ) +} diff --git a/cli/disk_cache.rs b/cli/disk_cache.rs index 828ca90ca615a9..58744acd8fea8a 100644 --- a/cli/disk_cache.rs +++ b/cli/disk_cache.rs @@ -31,14 +31,14 @@ impl DiskCache { } /// Ensures the location of the cache. - pub fn ensure_location(&self) -> io::Result<()> { - if self.location.is_dir() { + pub fn ensure_dir_exists(&self, path: &Path) -> io::Result<()> { + if path.is_dir() { return Ok(()); } - fs::create_dir_all(&self.location).map_err(|e| { + fs::create_dir_all(&path).map_err(|e| { io::Error::new(e.kind(), format!( "Could not create TypeScript compiler cache location: {:?}\nCheck the permission of the directory.", - self.location + path )) }) } @@ -129,8 +129,7 @@ impl DiskCache { pub fn set(&self, filename: &Path, data: &[u8]) -> std::io::Result<()> { let path = self.location.join(filename); match path.parent() { - Some(ref parent) => fs::create_dir_all(parent) - .map_err(|e| with_io_context(&e, format!("{:#?}", &path))), + Some(ref parent) => self.ensure_dir_exists(parent), None => Ok(()), }?; deno_fs::write_file(&path, data, 0o666) @@ -154,7 +153,9 @@ mod tests { let mut cache_path = cache_location.path().to_owned(); cache_path.push("foo"); let cache = DiskCache::new(&cache_path); - cache.ensure_location().expect("Testing expect:"); + cache + .ensure_dir_exists(&cache.location) + .expect("Testing expect:"); assert!(cache_path.is_dir()); } @@ -166,7 +167,9 @@ mod tests { cache_location.push("foo"); assert_eq!(cache_location.is_dir(), false); let cache = DiskCache::new(&cache_location); - cache.ensure_location().expect("Testing expect:"); + cache + .ensure_dir_exists(&cache.location) + .expect("Testing expect:"); assert_eq!(cache_location.is_dir(), true); } diff --git a/cli/doc/node.rs b/cli/doc/node.rs index 0846db0dd82a00..0f97ed65abe734 100644 --- a/cli/doc/node.rs +++ b/cli/doc/node.rs @@ -32,7 +32,7 @@ pub struct ParamDef { pub ts_type: Option, } -#[derive(Debug, Serialize, Clone)] +#[derive(Debug, Serialize, Clone, PartialEq)] pub struct Location { pub filename: String, pub line: usize, diff --git a/cli/doc/parser.rs b/cli/doc/parser.rs index 0e58e4b0a655d8..3746e3dfba0c32 100644 --- a/cli/doc/parser.rs +++ b/cli/doc/parser.rs @@ -1,4 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::file_fetcher::map_file_extension; use crate::op_error::OpError; use crate::swc_common::comments::CommentKind; use crate::swc_common::Span; @@ -15,6 +16,7 @@ use deno_core::ModuleSpecifier; use futures::Future; use regex::Regex; use std::collections::HashMap; +use std::path::PathBuf; use std::pin::Pin; use super::namespace::NamespaceDef; @@ -57,9 +59,12 @@ impl DocParser { file_name: &str, source_code: &str, ) -> Result { - self - .ast_parser - .parse_module(file_name, source_code, |parse_result| { + let media_type = map_file_extension(&PathBuf::from(file_name)); + self.ast_parser.parse_module( + file_name, + media_type, + source_code, + |parse_result| { let module = parse_result?; let doc_entries = self.get_doc_nodes_for_module_body(module.body.clone()); @@ -69,7 +74,8 @@ impl DocParser { reexports, }; Ok(module_doc) - }) + }, + ) } pub async fn parse(&self, file_name: &str) -> Result, ErrBox> { @@ -524,7 +530,7 @@ impl DocParser { pub fn js_doc_for_span(&self, span: Span) -> Option { let comments = self.ast_parser.get_span_comments(span); - let js_doc_comment = comments.iter().find(|comment| { + let js_doc_comment = comments.iter().rev().find(|comment| { comment.kind == CommentKind::Block && comment.text.starts_with('*') })?; diff --git a/cli/doc/printer.rs b/cli/doc/printer.rs index f7f41079db349d..d24e659015f806 100644 --- a/cli/doc/printer.rs +++ b/cli/doc/printer.rs @@ -35,12 +35,13 @@ pub fn format_details(node: doc::DocNode) -> String { let js_doc = node.js_doc.clone(); if let Some(js_doc) = js_doc { - details.push_str(&format_jsdoc(js_doc, false, 1)); + details.push_str(&format_jsdoc(js_doc, 1)); } details.push_str("\n"); let maybe_extra = match node.kind { DocNodeKind::Class => Some(format_class_details(node)), + DocNodeKind::Enum => Some(format_enum_details(node)), DocNodeKind::Namespace => Some(format_namespace_details(node)), _ => None, }; @@ -92,7 +93,7 @@ fn format_(doc_nodes: Vec, indent: i64) -> String { for node in sorted { output.push_str(&format_signature(&node, indent)); if let Some(js_doc) = node.js_doc { - output.push_str(&format_jsdoc(js_doc, true, indent)); + output.push_str(&format_jsdoc(js_doc, indent)); } output.push_str("\n"); if DocNodeKind::Namespace == node.kind { @@ -308,19 +309,15 @@ fn add_indent(string: String, indent: i64) -> String { } // TODO: this should use some sort of markdown to console parser. -fn format_jsdoc(jsdoc: String, truncated: bool, indent: i64) -> String { - let mut lines = jsdoc.split("\n\n").map(|line| line.replace("\n", " ")); +fn format_jsdoc(jsdoc: String, indent: i64) -> String { + let lines = jsdoc.split("\n\n").map(|line| line.replace("\n", " ")); let mut js_doc = String::new(); - if truncated { - let first_line = lines.next().unwrap_or_else(|| "".to_string()); - js_doc.push_str(&add_indent(format!("{}\n", first_line), indent + 1)); - } else { - for line in lines { - js_doc.push_str(&add_indent(format!("{}\n", line), indent + 1)); - } + for line in lines { + js_doc.push_str(&add_indent(format!("{}\n", line), indent + 1)); } + format!("{}", colors::gray(js_doc)) } @@ -416,6 +413,17 @@ fn format_class_details(node: doc::DocNode) -> String { details } +fn format_enum_details(node: doc::DocNode) -> String { + let mut details = String::new(); + let enum_def = node.enum_def.unwrap(); + for member in enum_def.members { + details + .push_str(&add_indent(format!("{}\n", colors::bold(member.name)), 1)); + } + details.push_str("\n"); + details +} + fn format_namespace_details(node: doc::DocNode) -> String { let mut ns = String::new(); diff --git a/cli/doc/tests.rs b/cli/doc/tests.rs index 7f2c1863962bee..f3355a520c579c 100644 --- a/cli/doc/tests.rs +++ b/cli/doc/tests.rs @@ -44,6 +44,10 @@ impl DocFileLoader for TestLoader { #[tokio::test] async fn export_fn() { let source_code = r#"/** +* @module foo +*/ + +/** * Hello there, this is a multiline JSdoc. * * It has many lines @@ -51,6 +55,9 @@ async fn export_fn() { * Or not that many? */ export function foo(a: string, b?: number, cb: (...cbArgs: unknown[]) => void, ...args: unknown[]): void { + /** + * @todo document all the things. + */ console.log("Hello world"); } "#; @@ -143,7 +150,7 @@ export function foo(a: string, b?: number, cb: (...cbArgs: unknown[]) => void, . "location": { "col": 0, "filename": "test.ts", - "line": 8, + "line": 12, }, "name": "foo", }); @@ -877,6 +884,10 @@ export enum Hello { let actual = serde_json::to_value(entry).unwrap(); assert_eq!(actual, expected_json); + assert!(colors::strip_ansi_codes( + super::printer::format_details(entry.clone()).as_str() + ) + .contains("World")); assert!(colors::strip_ansi_codes( super::printer::format(entries.clone()).as_str() ) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 375142e0714476..be3bc6efc67053 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -544,6 +544,7 @@ pub fn map_file_extension(path: &Path) -> msg::MediaType { Some("js") => msg::MediaType::JavaScript, Some("jsx") => msg::MediaType::JSX, Some("mjs") => msg::MediaType::JavaScript, + Some("cjs") => msg::MediaType::JavaScript, Some("json") => msg::MediaType::Json, Some("wasm") => msg::MediaType::Wasm, _ => msg::MediaType::Unknown, @@ -572,7 +573,8 @@ fn map_content_type(path: &Path, content_type: Option<&str>) -> msg::MediaType { | "text/javascript" | "application/ecmascript" | "text/ecmascript" - | "application/x-javascript" => { + | "application/x-javascript" + | "application/node" => { map_js_like_extension(path, msg::MediaType::JavaScript) } "application/json" | "text/json" => msg::MediaType::Json, @@ -1596,6 +1598,10 @@ mod tests { map_file_extension(Path::new("foo/bar.wasm")), msg::MediaType::Wasm ); + assert_eq!( + map_file_extension(Path::new("foo/bar.cjs")), + msg::MediaType::JavaScript + ); assert_eq!( map_file_extension(Path::new("foo/bar.txt")), msg::MediaType::Unknown @@ -1641,6 +1647,10 @@ mod tests { map_content_type(Path::new("foo/bar.wasm"), None), msg::MediaType::Wasm ); + assert_eq!( + map_content_type(Path::new("foo/bar.cjs"), None), + msg::MediaType::JavaScript + ); assert_eq!( map_content_type(Path::new("foo/bar"), None), msg::MediaType::Unknown @@ -1694,6 +1704,10 @@ mod tests { map_content_type(Path::new("foo/bar"), Some("application/json")), msg::MediaType::Json ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("application/node")), + msg::MediaType::JavaScript + ); assert_eq!( map_content_type(Path::new("foo/bar"), Some("text/json")), msg::MediaType::Json diff --git a/cli/flags.rs b/cli/flags.rs index 075a29b5d475a0..fc16d378d95f86 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -1,5 +1,4 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -use crate::fs::resolve_from_cwd; use clap::App; use clap::AppSettings; use clap::Arg; @@ -7,7 +6,7 @@ use clap::ArgMatches; use clap::SubCommand; use log::Level; use std::net::SocketAddr; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; /// Creates vector of strings, Vec macro_rules! svec { @@ -29,6 +28,7 @@ pub enum DenoSubcommand { filter: Option, }, Eval { + print: bool, code: String, as_typescript: bool, }, @@ -50,6 +50,9 @@ pub enum DenoSubcommand { root: Option, force: bool, }, + Lint { + files: Vec, + }, Repl, Run { script: String, @@ -180,7 +183,8 @@ static ENV_VARIABLES_HELP: &str = "ENVIRONMENT VARIABLES: NO_COLOR Set to disable color HTTP_PROXY Proxy address for HTTP requests (module downloads, fetch) - HTTPS_PROXY Same but for HTTPS"; + HTTPS_PROXY Proxy address for HTTPS requests + (module downloads, fetch)"; static DENO_HELP: &str = "A secure JavaScript and TypeScript runtime @@ -260,6 +264,8 @@ pub fn flags_from_vec_safe(args: Vec) -> clap::Result { upgrade_parse(&mut flags, m); } else if let Some(m) = matches.subcommand_matches("doc") { doc_parse(&mut flags, m); + } else if let Some(m) = matches.subcommand_matches("lint") { + lint_parse(&mut flags, m); } else { repl_parse(&mut flags, &matches); } @@ -302,18 +308,19 @@ If the flag is set, restrict these messages to errors.", .global(true), ) .subcommand(bundle_subcommand()) + .subcommand(cache_subcommand()) .subcommand(completions_subcommand()) + .subcommand(doc_subcommand()) .subcommand(eval_subcommand()) - .subcommand(cache_subcommand()) .subcommand(fmt_subcommand()) .subcommand(info_subcommand()) .subcommand(install_subcommand()) + .subcommand(lint_subcommand()) .subcommand(repl_subcommand()) .subcommand(run_subcommand()) .subcommand(test_subcommand()) .subcommand(types_subcommand()) .subcommand(upgrade_subcommand()) - .subcommand(doc_subcommand()) .long_about(DENO_HELP) .after_help(ENV_VARIABLES_HELP) } @@ -431,7 +438,9 @@ fn eval_parse(flags: &mut Flags, matches: &clap::ArgMatches) { flags.allow_hrtime = true; let code = matches.value_of("code").unwrap().to_string(); let as_typescript = matches.is_present("ts"); + let print = matches.is_present("print"); flags.subcommand = DenoSubcommand::Eval { + print, code, as_typescript, } @@ -472,13 +481,6 @@ fn lock_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) { } } -fn resolve_fs_whitelist(whitelist: &[PathBuf]) -> Vec { - whitelist - .iter() - .map(|raw_path| resolve_from_cwd(Path::new(&raw_path)).unwrap()) - .collect() -} - // Shared between the run and test subcommands. They both take similar options. fn run_test_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) { reload_arg_parse(flags, matches); @@ -584,6 +586,16 @@ fn doc_parse(flags: &mut Flags, matches: &clap::ArgMatches) { }; } +fn lint_parse(flags: &mut Flags, matches: &clap::ArgMatches) { + unstable_arg_parse(flags, matches); + let files = matches + .values_of("files") + .unwrap() + .map(String::from) + .collect(); + flags.subcommand = DenoSubcommand::Lint { files }; +} + fn types_subcommand<'a, 'b>() -> App<'a, 'b> { SubCommand::with_name("types") .arg(unstable_arg()) @@ -752,6 +764,14 @@ This command has implicit access to all permissions (--allow-all).", .takes_value(false) .multiple(false), ) + .arg( + Arg::with_name("print") + .long("print") + .short("p") + .help("print result to stdout") + .takes_value(false) + .multiple(false), + ) .arg(Arg::with_name("code").takes_value(true).required(true)) .arg(v8_flags_arg()) } @@ -886,6 +906,25 @@ Show documentation for runtime built-ins: ) } +fn lint_subcommand<'a, 'b>() -> App<'a, 'b> { + SubCommand::with_name("lint") + .about("Lint source files") + .long_about( + "Lint JavaScript/TypeScript source code. + deno lint myfile1.ts myfile2.js + +Ignore diagnostics on next line preceding it with an ignore comment and code: + // deno-lint-ignore no-explicit-any", + ) + .arg(unstable_arg()) + .arg( + Arg::with_name("files") + .takes_value(true) + .required(true) + .min_values(1), + ) +} + fn permission_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> { app .arg( @@ -1030,7 +1069,7 @@ report results to standard output: deno test src/fetch_test.ts src/signal_test.ts Directory arguments are expanded to all contained files matching the glob -{*_,*.,}test.{js,ts,jsx,tsx}: +{*_,*.,}test.{js,mjs,ts,jsx,tsx}: deno test src/", ) } @@ -1235,25 +1274,22 @@ fn no_remote_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) { fn permission_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) { if let Some(read_wl) = matches.values_of("allow-read") { - let raw_read_whitelist: Vec = read_wl.map(PathBuf::from).collect(); + let read_whitelist: Vec = read_wl.map(PathBuf::from).collect(); - if raw_read_whitelist.is_empty() { + if read_whitelist.is_empty() { flags.allow_read = true; } else { - flags.read_whitelist = resolve_fs_whitelist(&raw_read_whitelist); - debug!("read whitelist: {:#?}", &flags.read_whitelist); + flags.read_whitelist = read_whitelist; } } if let Some(write_wl) = matches.values_of("allow-write") { - let raw_write_whitelist: Vec = - write_wl.map(PathBuf::from).collect(); + let write_whitelist: Vec = write_wl.map(PathBuf::from).collect(); - if raw_write_whitelist.is_empty() { + if write_whitelist.is_empty() { flags.allow_write = true; } else { - flags.write_whitelist = resolve_fs_whitelist(&raw_write_whitelist); - debug!("write whitelist: {:#?}", &flags.write_whitelist); + flags.write_whitelist = write_whitelist; } } @@ -1352,7 +1388,6 @@ fn resolve_hosts(paths: Vec) -> Vec { #[cfg(test)] mod tests { use super::*; - use std::env::current_dir; #[test] fn upgrade() { @@ -1705,6 +1740,7 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Eval { + print: false, code: "'console.log(\"hello\")'".to_string(), as_typescript: false, }, @@ -1720,6 +1756,29 @@ mod tests { ); } + #[test] + fn eval_p() { + let r = flags_from_vec_safe(svec!["deno", "eval", "-p", "1+2"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Eval { + print: true, + code: "1+2".to_string(), + as_typescript: false, + }, + allow_net: true, + allow_env: true, + allow_run: true, + allow_read: true, + allow_write: true, + allow_plugin: true, + allow_hrtime: true, + ..Flags::default() + } + ); + } + #[test] fn eval_unstable() { let r = flags_from_vec_safe(svec![ @@ -1733,6 +1792,7 @@ mod tests { Flags { unstable: true, subcommand: DenoSubcommand::Eval { + print: false, code: "'console.log(\"hello\")'".to_string(), as_typescript: false, }, @@ -1760,6 +1820,7 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Eval { + print: false, code: "'console.log(\"hello\")'".to_string(), as_typescript: true, }, @@ -1783,6 +1844,7 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Eval { + print: false, code: "42".to_string(), as_typescript: false, }, @@ -1853,7 +1915,7 @@ mod tests { r.unwrap(), Flags { allow_read: false, - read_whitelist: vec![current_dir().unwrap(), temp_dir], + read_whitelist: vec![PathBuf::from("."), temp_dir], subcommand: DenoSubcommand::Run { script: "script.ts".to_string(), }, @@ -1877,7 +1939,7 @@ mod tests { r.unwrap(), Flags { allow_write: false, - write_whitelist: vec![current_dir().unwrap(), temp_dir], + write_whitelist: vec![PathBuf::from("."), temp_dir], subcommand: DenoSubcommand::Run { script: "script.ts".to_string(), }, @@ -2468,6 +2530,7 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Eval { + print: false, code: "console.log('hello world')".to_string(), as_typescript: false, }, @@ -2496,6 +2559,7 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Eval { + print: false, code: "const foo = 'bar'".to_string(), as_typescript: false, }, diff --git a/cli/fmt.rs b/cli/fmt.rs index 9b6e5975ddb079..0d7a909bc0cd77 100644 --- a/cli/fmt.rs +++ b/cli/fmt.rs @@ -7,6 +7,8 @@ //! the future it can be easily extended to provide //! the same functions as ops available in JS runtime. +use crate::colors; +use crate::diff::diff; use crate::fs::files_in_subtree; use crate::op_error::OpError; use deno_core::ErrBox; @@ -21,6 +23,8 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; +const BOM_CHAR: char = '\u{FEFF}'; + /// Format JavaScript/TypeScript files. /// /// First argument supports globs, and if it is `None` @@ -61,17 +65,38 @@ async fn check_source_files( ) -> Result<(), ErrBox> { let not_formatted_files_count = Arc::new(AtomicUsize::new(0)); let formatter = Arc::new(dprint::Formatter::new(config)); - let output_lock = Arc::new(Mutex::new(0)); // prevent threads outputting at the same time + + // prevent threads outputting at the same time + let output_lock = Arc::new(Mutex::new(0)); run_parallelized(paths, { let not_formatted_files_count = not_formatted_files_count.clone(); move |file_path| { - let file_contents = fs::read_to_string(&file_path)?; - let r = formatter.format_text(&file_path, &file_contents); + let file_text = read_file_contents(&file_path)?.text; + let r = formatter.format_text(&file_path, &file_text); match r { Ok(formatted_text) => { - if formatted_text != file_contents { + if formatted_text != file_text { not_formatted_files_count.fetch_add(1, Ordering::SeqCst); + let _g = output_lock.lock().unwrap(); + match diff(&file_text, &formatted_text) { + Ok(diff) => { + println!(); + println!( + "{} {}:", + colors::bold("from".to_string()), + file_path.display().to_string() + ); + println!("{}", diff); + } + Err(e) => { + eprintln!( + "Error generating diff: {}", + file_path.to_string_lossy() + ); + eprintln!(" {}", e); + } + } } } Err(e) => { @@ -112,12 +137,18 @@ async fn format_source_files( run_parallelized(paths, { let formatted_files_count = formatted_files_count.clone(); move |file_path| { - let file_contents = fs::read_to_string(&file_path)?; - let r = formatter.format_text(&file_path, &file_contents); + let file_contents = read_file_contents(&file_path)?; + let r = formatter.format_text(&file_path, &file_contents.text); match r { Ok(formatted_text) => { - if formatted_text != file_contents { - fs::write(&file_path, formatted_text)?; + if formatted_text != file_contents.text { + write_file_contents( + &file_path, + FileContents { + had_bom: file_contents.had_bom, + text: formatted_text, + }, + )?; formatted_files_count.fetch_add(1, Ordering::SeqCst); let _g = output_lock.lock().unwrap(); println!("{}", file_path.to_string_lossy()); @@ -171,13 +202,6 @@ fn format_stdin(check: bool) -> Result<(), ErrBox> { Ok(()) } -/// Formats the given source text -pub fn format_text(source: &str) -> Result { - dprint::Formatter::new(get_config()) - .format_text(&PathBuf::from("_tmp.ts"), &source) - .map_err(|e| OpError::other(e).into()) -} - fn files_str(len: usize) -> &'static str { if len == 1 { "file" @@ -192,7 +216,7 @@ fn is_supported(path: &Path) -> bool { .and_then(|e| e.to_str()) .map(|e| e.to_lowercase()); if let Some(ext) = lowercase_ext { - ext == "ts" || ext == "tsx" || ext == "js" || ext == "jsx" + ext == "ts" || ext == "tsx" || ext == "js" || ext == "jsx" || ext == "mjs" } else { false } @@ -203,6 +227,38 @@ fn get_config() -> dprint::configuration::Configuration { ConfigurationBuilder::new().deno().build() } +struct FileContents { + text: String, + had_bom: bool, +} + +fn read_file_contents(file_path: &PathBuf) -> Result { + let file_text = fs::read_to_string(&file_path)?; + let had_bom = file_text.starts_with(BOM_CHAR); + let text = if had_bom { + // remove the BOM + String::from(&file_text[BOM_CHAR.len_utf8()..]) + } else { + file_text + }; + + Ok(FileContents { text, had_bom }) +} + +fn write_file_contents( + file_path: &PathBuf, + file_contents: FileContents, +) -> Result<(), ErrBox> { + let file_text = if file_contents.had_bom { + // add back the BOM + format!("{}{}", BOM_CHAR, file_contents.text) + } else { + file_contents.text + }; + + Ok(fs::write(file_path, file_text)?) +} + async fn run_parallelized( file_paths: Vec, f: F, @@ -260,6 +316,8 @@ fn test_is_supported() { assert!(is_supported(Path::new("foo.TSX"))); assert!(is_supported(Path::new("foo.JS"))); assert!(is_supported(Path::new("foo.JSX"))); + assert!(is_supported(Path::new("foo.mjs"))); + assert!(!is_supported(Path::new("foo.mjsx"))); } #[tokio::test] diff --git a/cli/fmt_errors.rs b/cli/fmt_errors.rs index c8dfc625a2b5b9..bfda1feb0d8cba 100644 --- a/cli/fmt_errors.rs +++ b/cli/fmt_errors.rs @@ -10,6 +10,15 @@ use std::ops::Deref; const SOURCE_ABBREV_THRESHOLD: usize = 150; +pub fn format_location(filename: String, line: i64, col: i64) -> String { + format!( + "{}:{}:{}", + colors::cyan(filename), + colors::yellow(line.to_string()), + colors::yellow(col.to_string()) + ) +} + pub fn format_stack( is_error: bool, message_line: String, @@ -137,11 +146,10 @@ impl fmt::Display for JSError { && self.0.line_number.is_some() && self.0.start_column.is_some() { - formatted_frames = vec![format!( - "{}:{}:{}", - colors::cyan(self.0.script_resource_name.clone().unwrap()), - colors::yellow(self.0.line_number.unwrap().to_string()), - colors::yellow((self.0.start_column.unwrap() + 1).to_string()) + formatted_frames = vec![format_location( + self.0.script_resource_name.clone().unwrap(), + self.0.line_number.unwrap(), + self.0.start_column.unwrap() + 1, )] }; diff --git a/cli/global_state.rs b/cli/global_state.rs index 90961ed6271a8c..c1ed3af61fff33 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -3,9 +3,14 @@ use crate::deno_dir; use crate::file_fetcher::SourceFileFetcher; use crate::flags; use crate::http_cache; +use crate::import_map::ImportMap; use crate::lockfile::Lockfile; +use crate::module_graph::ModuleGraphFile; +use crate::module_graph::ModuleGraphLoader; use crate::msg; +use crate::msg::MediaType; use crate::permissions::Permissions; +use crate::state::exit_unstable; use crate::tsc::CompiledModule; use crate::tsc::TargetLib; use crate::tsc::TsCompiler; @@ -35,6 +40,7 @@ pub struct GlobalStateInner { pub ts_compiler: TsCompiler, pub lockfile: Option>, pub compiler_starts: AtomicUsize, + pub maybe_import_map: Option, compile_lock: AsyncMutex<()>, } @@ -51,7 +57,6 @@ impl GlobalState { let dir = deno_dir::DenoDir::new(custom_root)?; let deps_cache_location = dir.root.join("deps"); let http_cache = http_cache::HttpCache::new(&deps_cache_location); - http_cache.ensure_location()?; let file_fetcher = SourceFileFetcher::new( http_cache, @@ -76,6 +81,17 @@ impl GlobalState { None }; + let maybe_import_map: Option = + match flags.import_map_path.as_ref() { + None => None, + Some(file_path) => { + if !flags.unstable { + exit_unstable("--importmap") + } + Some(ImportMap::load(file_path)?) + } + }; + let inner = GlobalStateInner { dir, permissions: Permissions::from_flags(&flags), @@ -83,19 +99,83 @@ impl GlobalState { file_fetcher, ts_compiler, lockfile, + maybe_import_map, compiler_starts: AtomicUsize::new(0), compile_lock: AsyncMutex::new(()), }; Ok(GlobalState(Arc::new(inner))) } - pub async fn fetch_compiled_module( + /// This function is called when new module load is + /// initialized by the EsIsolate. Its resposibility is to collect + /// all dependencies and if it is required then also perform TS typecheck + /// and traspilation. + pub async fn prepare_module_load( &self, module_specifier: ModuleSpecifier, maybe_referrer: Option, target_lib: TargetLib, permissions: Permissions, is_dyn_import: bool, + maybe_import_map: Option, + ) -> Result<(), ErrBox> { + let module_specifier = module_specifier.clone(); + + // TODO(ry) Try to lift compile_lock as high up in the call stack for + // sanity. + let compile_lock = self.compile_lock.lock().await; + + let mut module_graph_loader = ModuleGraphLoader::new( + self.file_fetcher.clone(), + maybe_import_map, + permissions.clone(), + is_dyn_import, + false, + ); + module_graph_loader + .add_to_graph(&module_specifier, maybe_referrer) + .await?; + let module_graph = module_graph_loader.get_graph(); + + let out = self + .file_fetcher + .fetch_cached_source_file(&module_specifier, permissions.clone()) + .expect("Source file not found"); + + // Check if we need to compile files. + let should_compile = needs_compilation( + self.ts_compiler.compile_js, + out.media_type, + module_graph.values().collect::>(), + ); + + if should_compile { + self + .ts_compiler + .compile_module_graph( + self.clone(), + &out, + target_lib, + permissions, + module_graph, + ) + .await?; + } + + drop(compile_lock); + + Ok(()) + } + + // TODO(bartlomieju): this method doesn't need to be async anymore + /// This method is used after `prepare_module_load` finishes and EsIsolate + /// starts loading source and executing source code. This method shouldn't + /// perform any IO (besides $DENO_DIR) and only operate on sources collected + /// during `prepare_module_load`. + pub async fn fetch_compiled_module( + &self, + module_specifier: ModuleSpecifier, + _maybe_referrer: Option, ) -> Result { let state1 = self.clone(); let state2 = self.clone(); @@ -103,59 +183,31 @@ impl GlobalState { let out = self .file_fetcher - .fetch_source_file(&module_specifier, maybe_referrer, permissions.clone()) - .await?; + .fetch_cached_source_file(&module_specifier, Permissions::allow_all()) + .expect("Cached source file doesn't exist"); // TODO(ry) Try to lift compile_lock as high up in the call stack for // sanity. let compile_lock = self.compile_lock.lock().await; - let compiled_module = match out.media_type { + // Check if we need to compile files + let was_compiled = match out.media_type { msg::MediaType::TypeScript | msg::MediaType::TSX - | msg::MediaType::JSX => { - state1 - .ts_compiler - .compile(state1.clone(), &out, target_lib, permissions, is_dyn_import) - .await - } - msg::MediaType::JavaScript => { - if state1.ts_compiler.compile_js { - state2 - .ts_compiler - .compile( - state1.clone(), - &out, - target_lib, - permissions, - is_dyn_import, - ) - .await - } else { - if let Some(types_url) = out.types_url.clone() { - let types_specifier = ModuleSpecifier::from(types_url); - state1 - .file_fetcher - .fetch_source_file( - &types_specifier, - Some(module_specifier.clone()), - permissions.clone(), - ) - .await - .ok(); - }; - - Ok(CompiledModule { - code: String::from_utf8(out.source_code.clone())?, - name: out.url.to_string(), - }) - } - } - _ => Ok(CompiledModule { + | msg::MediaType::JSX => true, + msg::MediaType::JavaScript => self.ts_compiler.compile_js, + _ => false, + }; + + let compiled_module = if was_compiled { + state1.ts_compiler.get_compiled_module(&out.url)? + } else { + CompiledModule { code: String::from_utf8(out.source_code.clone())?, name: out.url.to_string(), - }), - }?; + } + }; + drop(compile_lock); if let Some(ref lockfile) = state2.lockfile { @@ -189,6 +241,34 @@ impl GlobalState { } } +// Compilation happens if either: +// - `checkJs` is set to true in TS config +// - entry point is a TS file +// - any dependency in module graph is a TS file +fn needs_compilation( + compile_js: bool, + media_type: MediaType, + module_graph_files: Vec<&ModuleGraphFile>, +) -> bool { + let mut needs_compilation = match media_type { + msg::MediaType::TypeScript | msg::MediaType::TSX | msg::MediaType::JSX => { + true + } + msg::MediaType::JavaScript => compile_js, + _ => false, + }; + + needs_compilation |= module_graph_files.iter().any(|module_file| { + let media_type = module_file.media_type; + + media_type == (MediaType::TypeScript as i32) + || media_type == (MediaType::TSX as i32) + || media_type == (MediaType::JSX as i32) + }); + + needs_compilation +} + #[test] fn thread_safe() { fn f(_: S) {} @@ -196,9 +276,44 @@ fn thread_safe() { } #[test] -fn import_map_given_for_repl() { - let _result = GlobalState::new(flags::Flags { - import_map_path: Some("import_map.json".to_string()), - ..flags::Flags::default() - }); +fn test_needs_compilation() { + assert!(!needs_compilation( + false, + MediaType::JavaScript, + vec![&ModuleGraphFile { + specifier: "some/file.js".to_string(), + url: "file:///some/file.js".to_string(), + redirect: None, + filename: "some/file.js".to_string(), + imports: vec![], + referenced_files: vec![], + lib_directives: vec![], + types_directives: vec![], + type_headers: vec![], + media_type: MediaType::JavaScript as i32, + source_code: "function foo() {}".to_string(), + }] + )); + assert!(!needs_compilation(false, MediaType::JavaScript, vec![])); + assert!(needs_compilation(true, MediaType::JavaScript, vec![])); + assert!(needs_compilation(false, MediaType::TypeScript, vec![])); + assert!(needs_compilation(false, MediaType::JSX, vec![])); + assert!(needs_compilation(false, MediaType::TSX, vec![])); + assert!(needs_compilation( + false, + MediaType::JavaScript, + vec![&ModuleGraphFile { + specifier: "some/file.ts".to_string(), + url: "file:///some/file.ts".to_string(), + redirect: None, + filename: "some/file.ts".to_string(), + imports: vec![], + referenced_files: vec![], + lib_directives: vec![], + types_directives: vec![], + type_headers: vec![], + media_type: MediaType::TypeScript as i32, + source_code: "function foo() {}".to_string(), + }] + )); } diff --git a/cli/http_cache.rs b/cli/http_cache.rs index 2a9882376a0049..23b482840d68bb 100644 --- a/cli/http_cache.rs +++ b/cli/http_cache.rs @@ -113,16 +113,16 @@ impl HttpCache { } /// Ensures the location of the cache. - pub fn ensure_location(&self) -> io::Result<()> { - if self.location.is_dir() { + fn ensure_dir_exists(&self, path: &Path) -> io::Result<()> { + if path.is_dir() { return Ok(()); } - fs::create_dir_all(&self.location).map_err(|e| { + fs::create_dir_all(&path).map_err(|e| { io::Error::new( e.kind(), format!( "Could not create remote modules cache location: {:?}\nCheck the permission of the directory.", - self.location + path ), ) }) @@ -163,7 +163,7 @@ impl HttpCache { let parent_filename = cache_filename .parent() .expect("Cache filename should have a parent dir"); - fs::create_dir_all(parent_filename)?; + self.ensure_dir_exists(parent_filename)?; // Cache content deno_fs::write_file(&cache_filename, content, 0o666)?; @@ -187,8 +187,25 @@ mod tests { let dir = TempDir::new().unwrap(); let mut cache_path = dir.path().to_owned(); cache_path.push("foobar"); + // HttpCache should be created lazily on first use: + // when zipping up a local project with no external dependencies + // "$DENO_DIR/deps" is empty. When unzipping such project + // "$DENO_DIR/deps" might not get restored and in situation + // when directory is owned by root we might not be able + // to create that directory. However if it's not needed it + // doesn't make sense to return error in such specific scenarios. + // For more details check issue: + // https://github.com/denoland/deno/issues/5688 let cache = HttpCache::new(&cache_path); - assert!(cache.ensure_location().is_ok()); + assert!(!cache.location.exists()); + cache + .set( + &Url::parse("http://example.com/foo/bar.js").unwrap(), + HeadersMap::new(), + b"hello world", + ) + .expect("Failed to add to cache"); + assert!(cache.ensure_dir_exists(&cache.location).is_ok()); assert!(cache_path.is_dir()); } diff --git a/cli/inspector.rs b/cli/inspector.rs index b67bb89f5a1c0e..94114addb47611 100644 --- a/cli/inspector.rs +++ b/cli/inspector.rs @@ -375,14 +375,10 @@ impl DenoInspector { isolate: &mut deno_core::CoreIsolate, host: SocketAddr, ) -> Box { - let deno_core::CoreIsolate { - v8_isolate, - global_context, - .. - } = isolate; - - let v8_isolate = v8_isolate.as_mut().unwrap(); - let mut hs = v8::HandleScope::new(v8_isolate); + let core_state_rc = deno_core::CoreIsolate::state(isolate); + let core_state = core_state_rc.borrow(); + + let mut hs = v8::HandleScope::new(isolate); let scope = hs.enter(); let (new_websocket_tx, new_websocket_rx) = @@ -420,7 +416,7 @@ impl DenoInspector { }); // Tell the inspector about the global context. - let context = global_context.get(scope).unwrap(); + let context = core_state.global_context.get(scope).unwrap(); let context_name = v8::inspector::StringView::from(&b"global context"[..]); self_.context_created(context, Self::CONTEXT_GROUP_ID, context_name); diff --git a/cli/installer.rs b/cli/installer.rs index 57caa3a3dc9982..a6f21f10c90f88 100644 --- a/cli/installer.rs +++ b/cli/installer.rs @@ -120,22 +120,18 @@ fn get_installer_root() -> Result { return PathBuf::from(env_dir).canonicalize(); } } - // In Windows's Powershell $HOME environmental variable maybe null - // if so use $USERPROFILE instead. - let home = env::var("HOME") - .map(String::into) - .unwrap_or_else(|_| "".to_string()); - let user_profile = env::var("USERPROFILE") - .map(String::into) - .unwrap_or_else(|_| "".to_string()); - - if home.is_empty() && user_profile.is_empty() { - return Err(Error::new(ErrorKind::Other, "$HOME is not defined")); - } - - let home_path = if !home.is_empty() { home } else { user_profile }; - - let mut home_path = PathBuf::from(home_path); + // Note: on Windows, the $HOME environment variable may be set by users or by + // third party software, but it is non-standard and should not be relied upon. + let home_env_var = if cfg!(windows) { "USERPROFILE" } else { "HOME" }; + let mut home_path = + env::var_os(home_env_var) + .map(PathBuf::from) + .ok_or_else(|| { + Error::new( + ErrorKind::NotFound, + format!("${} is not defined", home_env_var), + ) + })?; home_path.push(".deno"); Ok(home_path) } diff --git a/cli/js/diagnostics_util.ts b/cli/js/diagnostics_util.ts index 17e73e377da212..7b66d72a39557e 100644 --- a/cli/js/diagnostics_util.ts +++ b/cli/js/diagnostics_util.ts @@ -10,6 +10,79 @@ import { DiagnosticItem, } from "./diagnostics.ts"; +const unstableDenoGlobalProperties = [ + "umask", + "linkSync", + "link", + "symlinkSync", + "symlink", + "DirKind", + "dir", + "loadavg", + "osRelease", + "openPlugin", + "DiagnosticCategory", + "DiagnosticMessageChain", + "DiagnosticItem", + "Diagnostic", + "formatDiagnostics", + "CompilerOptions", + "TranspileOnlyResult", + "transpileOnly", + "compile", + "bundle", + "Location", + "applySourceMap", + "LinuxSignal", + "MacOSSignal", + "Signal", + "SignalStream", + "signal", + "signals", + "setRaw", + "utimeSync", + "utime", + "ShutdownMode", + "shutdown", + "DatagramConn", + "UnixListenOptions", + "listen", + "listenDatagram", + "UnixConnectOptions", + "connect", + "StartTlsOptions", + "startTls", + "kill", + "PermissionName", + "PermissionState", + "RunPermissionDescriptor", + "ReadPermissionDescriptor", + "WritePermissionDescriptor", + "NetPermissionDescriptor", + "EnvPermissionDescriptor", + "PluginPermissionDescriptor", + "HrtimePermissionDescriptor", + "PermissionDescriptor", + "Permissions", + "PermissionStatus", + "hostname", +]; + +function transformMessageText(messageText: string, code: number): string { + if (code === 2339) { + const property = messageText + .replace(/^Property '/, "") + .replace(/' does not exist on type 'typeof Deno'\.$/, ""); + if ( + messageText.endsWith("on type 'typeof Deno'.") && + unstableDenoGlobalProperties.includes(property) + ) { + return `${messageText} 'Deno.${property}' is an unstable API. Did you forget to run with the '--unstable' flag?`; + } + } + return messageText; +} + interface SourceInformation { sourceLine: string; lineNumber: number; @@ -78,7 +151,8 @@ function fromDiagnosticMessageChain( return undefined; } - return messageChain.map(({ messageText: message, code, category, next }) => { + return messageChain.map(({ messageText, code, category, next }) => { + const message = transformMessageText(messageText, code); return { message, code, @@ -110,9 +184,9 @@ function parseDiagnostic( let message: string; let messageChain: DiagnosticMessageChain | undefined; if (typeof messageText === "string") { - message = messageText; + message = transformMessageText(messageText, code); } else { - message = messageText.messageText; + message = transformMessageText(messageText.messageText, messageText.code); messageChain = fromDiagnosticMessageChain([messageText])![0]; } diff --git a/cli/js/error_stack.ts b/cli/js/error_stack.ts index e77f0865c995e0..39561ad854d1bc 100644 --- a/cli/js/error_stack.ts +++ b/cli/js/error_stack.ts @@ -215,50 +215,47 @@ function evaluateCallSite(callSite: CallSite): CallSiteEval { } function prepareStackTrace( - error: Error, - structuredStackTrace: CallSite[] + error: Error & { + __callSiteEvals: CallSiteEval[]; + __formattedFrames: string[]; + }, + callSites: CallSite[] ): string { + const mappedCallSites = callSites.map( + (callSite): CallSite => { + const fileName = callSite.getFileName(); + const lineNumber = callSite.getLineNumber(); + const columnNumber = callSite.getColumnNumber(); + if (fileName && lineNumber != null && columnNumber != null) { + return patchCallSite( + callSite, + applySourceMap({ + fileName, + lineNumber, + columnNumber, + }) + ); + } + return callSite; + } + ); Object.defineProperties(error, { - __callSiteEvals: { value: [] }, - __formattedFrames: { value: [] }, + __callSiteEvals: { value: [], configurable: true }, + __formattedFrames: { value: [], configurable: true }, }); - const errorString = - `${error.name}: ${error.message}\n` + - structuredStackTrace - .map( - (callSite): CallSite => { - const fileName = callSite.getFileName(); - const lineNumber = callSite.getLineNumber(); - const columnNumber = callSite.getColumnNumber(); - if (fileName && lineNumber != null && columnNumber != null) { - return patchCallSite( - callSite, - applySourceMap({ - fileName, - lineNumber, - columnNumber, - }) - ); - } - return callSite; - } - ) - .map((callSite): string => { - // @ts-ignore - error.__callSiteEvals.push(Object.freeze(evaluateCallSite(callSite))); - const isInternal = - callSite.getFileName()?.startsWith("$deno$") ?? false; - const string = callSiteToString(callSite, isInternal); - // @ts-ignore - error.__formattedFrames.push(string); - return ` at ${colors.stripColor(string)}`; - }) - .join("\n"); - // @ts-ignore + for (const callSite of mappedCallSites) { + error.__callSiteEvals.push(Object.freeze(evaluateCallSite(callSite))); + const isInternal = callSite.getFileName()?.startsWith("$deno$") ?? false; + error.__formattedFrames.push(callSiteToString(callSite, isInternal)); + } Object.freeze(error.__callSiteEvals); - // @ts-ignore Object.freeze(error.__formattedFrames); - return errorString; + return ( + `${error.name}: ${error.message}\n` + + error.__formattedFrames + .map((s: string) => ` at ${colors.stripColor(s)}`) + .join("\n") + ); } // @internal diff --git a/cli/js/globals.ts b/cli/js/globals.ts index 9908609be84443..09be63315b44a4 100644 --- a/cli/js/globals.ts +++ b/cli/js/globals.ts @@ -80,7 +80,7 @@ declare global { dispatch( opId: number, control: Uint8Array, - zeroCopy?: ArrayBufferView | null + ...zeroCopy: ArrayBufferView[] ): Uint8Array | null; setAsyncHandler(opId: number, cb: (msg: Uint8Array) => void): void; sharedQueue: { @@ -99,7 +99,7 @@ declare global { send( opId: number, control: null | ArrayBufferView, - data?: ArrayBufferView + ...data: ArrayBufferView[] ): null | Uint8Array; setMacrotaskCallback(cb: () => boolean): void; diff --git a/cli/js/lib.deno.ns.d.ts b/cli/js/lib.deno.ns.d.ts index 2415214a505d20..63e9b342ed89ee 100644 --- a/cli/js/lib.deno.ns.d.ts +++ b/cli/js/lib.deno.ns.d.ts @@ -364,7 +364,7 @@ declare namespace Deno { * * ```ts * let f = Deno.openSync("/etc/passwd"); - * for (const chunk of Deno.iterSync(reader)) { + * for (const chunk of Deno.iterSync(f)) { * console.log(chunk); * } * f.close(); @@ -509,7 +509,7 @@ declare namespace Deno { * ```ts * const encoder = new TextEncoder(); * const data = encoder.encode("Hello world"); - * const file = Deno.openSync("/foo/bar.txt"); + * const file = Deno.openSync("/foo/bar.txt", {write: true}); * const bytesWritten = Deno.writeSync(file.rid, data); // 11 * Deno.close(file.rid); * ``` @@ -528,7 +528,7 @@ declare namespace Deno { * ```ts * const encoder = new TextEncoder(); * const data = encoder.encode("Hello world"); - * const file = await Deno.open("/foo/bar.txt"); + * const file = await Deno.open("/foo/bar.txt", { write: true }); * const bytesWritten = await Deno.write(file.rid, data); // 11 * Deno.close(file.rid); * ``` @@ -897,7 +897,11 @@ declare namespace Deno { export interface MakeTempOptions { /** Directory where the temporary directory should be created (defaults to - * the env variable TMPDIR, or the system's default, usually /tmp). */ + * the env variable TMPDIR, or the system's default, usually /tmp). + * + * Note that if the passed `dir` is relative, the path returned by + * makeTempFile() and makeTempDir() will also be relative. Be mindful of + * this when changing working directory. */ dir?: string; /** String that should precede the random portion of the temporary * directory's name. */ @@ -1253,7 +1257,9 @@ declare namespace Deno { * console.log(realSymLinkPath); // outputs "/home/alice/file.txt" * ``` * - * Requires `allow-read` permission. */ + * Requires `allow-read` permission for the target path. + * Also requires `allow-read` permission for the CWD if the target path is + * relative.*/ export function realPathSync(path: string): string; /** Resolves to the absolute normalized path, with symbolic links resolved. @@ -1267,7 +1273,9 @@ declare namespace Deno { * console.log(realSymLinkPath); // outputs "/home/alice/file.txt" * ``` * - * Requires `allow-read` permission. */ + * Requires `allow-read` permission for the target path. + * Also requires `allow-read` permission for the CWD if the target path is + * relative.*/ export function realPath(path: string): Promise; export interface DirEntry { @@ -1358,6 +1366,7 @@ declare namespace Deno { * points to. * * ```ts + * import { assert } from "https://deno.land/std/testing/asserts.ts"; * const fileInfo = await Deno.lstat("hello.txt"); * assert(fileInfo.isFile); * ``` @@ -1381,6 +1390,7 @@ declare namespace Deno { * follow symlinks. * * ```ts + * import { assert } from "https://deno.land/std/testing/asserts.ts"; * const fileInfo = await Deno.stat("hello.txt"); * assert(fileInfo.isFile); * ``` @@ -1392,6 +1402,7 @@ declare namespace Deno { * always follow symlinks. * * ```ts + * import { assert } from "https://deno.land/std/testing/asserts.ts"; * const fileInfo = Deno.statSync("hello.txt"); * assert(fileInfo.isFile); * ``` @@ -1534,6 +1545,9 @@ declare namespace Deno { /** Return the address of the `Listener`. */ readonly addr: Addr; + /** Return the rid of the `Listener`. */ + readonly rid: number; + [Symbol.asyncIterator](): AsyncIterableIterator; } @@ -1612,10 +1626,9 @@ declare namespace Deno { * const conn2 = await Deno.connect({ hostname: "192.0.2.1", port: 80 }); * const conn3 = await Deno.connect({ hostname: "[2001:db8::1]", port: 80 }); * const conn4 = await Deno.connect({ hostname: "golang.org", port: 80, transport: "tcp" }); - * const conn5 = await Deno.connect({ path: "/foo/bar.sock", transport: "unix" }); * ``` * - * Requires `allow-net` permission for "tcp" and `allow-read` for unix. */ + * Requires `allow-net` permission for "tcp". */ export function connect(options: ConnectOptions): Promise; export interface ConnectTlsOptions { diff --git a/cli/js/lib.deno.shared_globals.d.ts b/cli/js/lib.deno.shared_globals.d.ts index ed1c1ac0b5dbdc..a93ff1166cdda1 100644 --- a/cli/js/lib.deno.shared_globals.d.ts +++ b/cli/js/lib.deno.shared_globals.d.ts @@ -174,34 +174,92 @@ declare namespace WebAssembly { } } -/** Sets a timer which executes a function once after the timer expires. */ +/** Sets a timer which executes a function once after the timer expires. Returns + * an id which may be used to cancel the timeout. + * + * setTimeout(() => { console.log('hello'); }, 500); + */ declare function setTimeout( + /** callback function to execute when timer expires */ cb: (...args: any[]) => void, + /** delay in ms */ delay?: number, + /** arguments passed to callback function */ ...args: any[] ): number; -/** Repeatedly calls a function , with a fixed time delay between each call. */ +/** Repeatedly calls a function , with a fixed time delay between each call. + * + * // Outputs 'hello' to the console every 500ms + * setInterval(() => { console.log('hello'); }, 500); + */ declare function setInterval( + /** callback function to execute when timer expires */ cb: (...args: any[]) => void, + /** delay in ms */ delay?: number, + /** arguments passed to callback function */ ...args: any[] ): number; -declare function clearTimeout(id?: number): void; + +/** Cancels a timed, repeating action which was previously started by a call + * to `setInterval()` + * + * const id = setInterval(()= > {console.log('hello');}, 500); + * ... + * clearInterval(id); + */ declare function clearInterval(id?: number): void; + +/** Cancels a scheduled action initiated by `setTimeout()` + * + * const id = setTimeout(()= > {console.log('hello');}, 500); + * ... + * clearTimeout(id); + */ +declare function clearTimeout(id?: number): void; + +/** A microtask is a short function which is executed after the function or + * module which created it exits and only if the JavaScript execution stack is + * empty, but before returning control to the event loop being used to drive the + * script's execution environment. This event loop may be either the main event + * loop or the event loop driving a web worker. + * + * queueMicrotask(() => { console.log('This event loop stack is complete'); }); + */ declare function queueMicrotask(func: Function): void; declare var console: Console; declare var crypto: Crypto; +/** Registers an event listener in the global scope, which will be called + * synchronously whenever the event `type` is dispatched. + * + * addEventListener('unload', () => { console.log('All finished!'); }); + * ... + * dispatchEvent(new Event('unload')); + */ declare function addEventListener( type: string, callback: EventListenerOrEventListenerObject | null, options?: boolean | AddEventListenerOptions | undefined ): void; +/** Dispatches an event in the global scope, synchronously invoking any + * registered event listeners for this event in the appropriate order. Returns + * false if event is cancelable and at least one of the event handlers which + * handled this event called Event.preventDefault(). Otherwise it returns true. + * + * dispatchEvent(new Event('unload')); + */ declare function dispatchEvent(event: Event): boolean; +/** Remove a previously registered event listener from the global scope + * + * const lstnr = () => { console.log('hello'); }; + * addEventListener('load', lstnr); + * removeEventListener('load', lstnr); + */ declare function removeEventListener( type: string, callback: EventListenerOrEventListenerObject | null, @@ -922,6 +980,12 @@ declare const Request: { new (input: RequestInfo, init?: RequestInit): Request; }; +interface ResponseInit { + headers?: HeadersInit; + status?: number; + statusText?: string; +} + type ResponseType = | "basic" | "cors" @@ -945,33 +1009,34 @@ interface Response extends Body { declare const Response: { prototype: Response; - - // TODO(#4667) Response constructor is non-standard. - // new(body?: BodyInit | null, init?: ResponseInit): Response; - new ( - url: string, - status: number, - statusText: string, - headersList: Array<[string, string]>, - rid: number, - redirected_: boolean, - type_?: null | ResponseType, - body_?: null | Body - ): Response; - + new (body?: BodyInit | null, init?: ResponseInit): Response; error(): Response; redirect(url: string, status?: number): Response; }; -/** Fetch a resource from the network. */ +/** Fetch a resource from the network. It returns a Promise that resolves to the + * Response to that request, whether it is successful or not. + * + * const response = await fetch("http://my.json.host/data.json"); + * console.log(response.status); // e.g. 200 + * console.log(response.statusText); // e.g. "OK" + * const jsonData = await response.json(); + */ declare function fetch( input: Request | URL | string, init?: RequestInit ): Promise; +/** Decodes a string of data which has been encoded using base-64 encoding. + * + * console.log(atob("aGVsbG8gd29ybGQ=")); // outputs 'hello world' + */ declare function atob(s: string): string; -/** Creates a base-64 ASCII string from the input string. */ +/** Creates a base-64 ASCII encoded string from the input string. + * + * console.log(btoa("hello world")); // outputs "aGVsbG8gd29ybGQ=" + */ declare function btoa(s: string): string; declare class TextDecoder { @@ -1273,7 +1338,7 @@ declare class Worker extends EventTarget { declare namespace performance { /** Returns a current time from Deno's start in milliseconds. * - * Use the flag --allow-hrtime return a precise value. + * Use the permission flag `--allow-hrtime` return a precise value. * * ```ts * const t = performance.now(); @@ -1488,3 +1553,11 @@ declare const AbortSignal: { prototype: AbortSignal; new (): AbortSignal; }; + +interface ErrorConstructor { + /** See https://v8.dev/docs/stack-trace-api#stack-trace-collection-for-custom-exceptions. */ + // eslint-disable-next-line @typescript-eslint/ban-types + captureStackTrace(error: Object, constructor?: Function): void; + // TODO(nayeemrmn): Support `Error.prepareStackTrace()`. We currently use this + // internally in a way that makes it unavailable for users. +} diff --git a/cli/js/lib.deno.unstable.d.ts b/cli/js/lib.deno.unstable.d.ts index 9bb4cfb90bb74d..aebd7f964142fb 100644 --- a/cli/js/lib.deno.unstable.d.ts +++ b/cli/js/lib.deno.unstable.d.ts @@ -21,7 +21,9 @@ declare namespace Deno { */ export function umask(mask?: number): number; - /** Synchronously creates `newpath` as a hard link to `oldpath`. + /** **UNSTABLE**: This API needs a security review. + * + * Synchronously creates `newpath` as a hard link to `oldpath`. * * ```ts * Deno.linkSync("old/name", "new/name"); @@ -30,9 +32,9 @@ declare namespace Deno { * Requires `allow-read` and `allow-write` permissions. */ export function linkSync(oldpath: string, newpath: string): void; - /** Creates `newpath` as a hard link to `oldpath`. + /** **UNSTABLE**: This API needs a security review. * - * **UNSTABLE**: needs security review. + * Creates `newpath` as a hard link to `oldpath`. * * ```ts * await Deno.link("old/name", "new/name"); @@ -45,7 +47,7 @@ declare namespace Deno { type: "file" | "dir"; }; - /** **UNSTABLE**: needs security review. + /** **UNSTABLE**: This API needs a security review. * * Creates `newpath` as a symbolic link to `oldpath`. * @@ -63,7 +65,7 @@ declare namespace Deno { options?: SymlinkOptions ): void; - /** **UNSTABLE**: needs security review. + /** **UNSTABLE**: This API needs a security review. * * Creates `newpath` as a symbolic link to `oldpath`. * @@ -250,7 +252,10 @@ declare namespace Deno { */ export function dir(kind: DirKind): string | null; - /** Returns an array containing the 1, 5, and 15 minute load averages. The + /** **Unstable** There are questions around which permission this needs. And + * maybe should be renamed (loadAverage?) + * + * Returns an array containing the 1, 5, and 15 minute load averages. The * load average is a measure of CPU and IO utilization of the last one, five, * and 15 minute periods expressed as a fractional number. Zero means there * is no load. On Windows, the three values are always the same and represent @@ -261,13 +266,14 @@ declare namespace Deno { * ``` * * Requires `allow-env` permission. - * - * **Unstable** There are questions around which permission this needs. And - * maybe should be renamed (loadAverage?) */ export function loadavg(): number[]; - /** Returns the release version of the Operating System. + /** **Unstable** new API. yet to be vetted. Under consideration to possibly move to + * Deno.build or Deno.versions and if it should depend sys-info, which may not + * be desireable. + * + * Returns the release version of the Operating System. * * ```ts * console.log(Deno.osRelease()); @@ -275,8 +281,6 @@ declare namespace Deno { * * Requires `allow-env` permission. * - * **Unstable** new API maybe move to Deno.build or Deno.versions? Depends on - * sys-info, which we don't necessarally want to depend on. */ export function osRelease(): string; @@ -699,7 +703,7 @@ declare namespace Deno { columnNumber: number; } - /** UNSTABLE: new API, yet to be vetted. + /** **UNSTABLE**: new API, yet to be vetted. * * Given a current location in a module, lookup the source location and return * it. @@ -793,7 +797,7 @@ declare namespace Deno { SIGUSR2 = 31, } - /** **UNSTABLE**: make platform independent. + /** **UNSTABLE**: Further changes required to make platform independent. * * Signals numbers. This is platform dependent. */ export const Signal: typeof MacOSSignal | typeof LinuxSignal; @@ -947,7 +951,7 @@ declare namespace Deno { mtime: number | Date ): Promise; - /** **UNSTABLE**: Maybe remove `ShutdownMode` entirely. + /** **UNSTABLE**: Under consideration to remove `ShutdownMode` entirely. * * Corresponds to `SHUT_RD`, `SHUT_WR`, `SHUT_RDWR` on POSIX-like systems. * @@ -973,7 +977,7 @@ declare namespace Deno { */ export function shutdown(rid: number, how: ShutdownMode): Promise; - /** **UNSTABLE**:: new API, yet to be vetted. + /** **UNSTABLE**: new API, yet to be vetted. * * A generic transport listener for message-oriented protocols. */ export interface DatagramConn extends AsyncIterable<[Uint8Array, Addr]> { @@ -1013,7 +1017,7 @@ declare namespace Deno { options: UnixListenOptions & { transport: "unix" } ): Listener; - /** **UNSTABLE**: new API + /** **UNSTABLE**: new API, yet to be vetted * * Listen announces on the local transport address. * @@ -1034,7 +1038,7 @@ declare namespace Deno { options: ListenOptions & { transport: "udp" } ): DatagramConn; - /** **UNSTABLE**: new API + /** **UNSTABLE**: new API, yet to be vetted * * Listen announces on the local transport address. * @@ -1055,7 +1059,9 @@ declare namespace Deno { path: string; } - /** + /** **UNSTABLE**: The unix socket transport is unstable as a new API yet to + * be vetted. The TCP transport is considered stable. + * * Connects to the hostname (default is "127.0.0.1") and port on the named * transport (default is "tcp"), and resolves to the connection (`Conn`). * @@ -1067,7 +1073,7 @@ declare namespace Deno { * const conn5 = await Deno.connect({ path: "/foo/bar.sock", transport: "unix" }); * ``` * - * Requires `allow-net` permission for "tcp" and `allow-read` for unix. */ + * Requires `allow-net` permission for "tcp" and `allow-read` for "unix". */ export function connect( options: ConnectOptions | UnixConnectOptions ): Promise; @@ -1216,8 +1222,8 @@ declare namespace Deno { request(desc: PermissionDescriptor): Promise; } - /** **UNSTABLE**: maybe move to `navigator.permissions` to match web API. It - * could look like `navigator.permissions.query({ name: Deno.symbols.read })`. + /** **UNSTABLE**: Under consideration to move to `navigator.permissions` to + * match web API. It could look like `navigator.permissions.query({ name: Deno.symbols.read })`. */ export const permissions: Permissions; @@ -1227,7 +1233,10 @@ declare namespace Deno { constructor(state: PermissionState); } - /** Get the `hostname` of the machine the Deno process is running on. + /** **UNSTABLE**: New API, yet to be vetted. Additional consideration is still + * necessary around the permissions required. + * + * Get the `hostname` of the machine the Deno process is running on. * * ```ts * console.log(Deno.hostname()); diff --git a/cli/js/net.ts b/cli/js/net.ts index 0c40cceae37a98..2de44c2acc13e8 100644 --- a/cli/js/net.ts +++ b/cli/js/net.ts @@ -26,6 +26,8 @@ export interface Listener extends AsyncIterable { addr: Addr; + rid: number; + [Symbol.asyncIterator](): AsyncIterableIterator; } diff --git a/cli/js/ops/dispatch_json.ts b/cli/js/ops/dispatch_json.ts index 9ff0f13f5980e4..6292c188a339ac 100644 --- a/cli/js/ops/dispatch_json.ts +++ b/cli/js/ops/dispatch_json.ts @@ -59,12 +59,12 @@ export function asyncMsgFromRust(resUi8: Uint8Array): void { export function sendSync( opName: string, args: object = {}, - zeroCopy?: Uint8Array + ...zeroCopy: Uint8Array[] ): Ok { const opId = OPS_CACHE[opName]; util.log("sendSync", opName, opId); const argsUi8 = encode(args); - const resUi8 = core.dispatch(opId, argsUi8, zeroCopy); + const resUi8 = core.dispatch(opId, argsUi8, ...zeroCopy); util.assert(resUi8 != null); const res = decode(resUi8); @@ -75,7 +75,7 @@ export function sendSync( export async function sendAsync( opName: string, args: object = {}, - zeroCopy?: Uint8Array + ...zeroCopy: Uint8Array[] ): Promise { const opId = OPS_CACHE[opName]; util.log("sendAsync", opName, opId); @@ -84,7 +84,7 @@ export async function sendAsync( const promise = util.createResolvable(); const argsUi8 = encode(args); - const buf = core.dispatch(opId, argsUi8, zeroCopy); + const buf = core.dispatch(opId, argsUi8, ...zeroCopy); if (buf) { // Sync result. const res = decode(buf); diff --git a/cli/js/ops/fetch.ts b/cli/js/ops/fetch.ts index 09b9ac1eca3e0a..290376c8604a07 100644 --- a/cli/js/ops/fetch.ts +++ b/cli/js/ops/fetch.ts @@ -24,5 +24,5 @@ export function fetch( zeroCopy = new Uint8Array(body.buffer, body.byteOffset, body.byteLength); } - return sendAsync("op_fetch", args, zeroCopy); + return sendAsync("op_fetch", args, ...(zeroCopy ? [zeroCopy] : [])); } diff --git a/cli/js/repl.ts b/cli/js/repl.ts index c84be68b1a973e..d1cf63cd918a85 100644 --- a/cli/js/repl.ts +++ b/cli/js/repl.ts @@ -35,8 +35,8 @@ function isRecoverableError(e: Error): boolean { // Returns `true` if `close()` is called in REPL. // We should quit the REPL when this function returns `true`. function isCloseCalled(): boolean { - // @ts-ignore - return globalThis.closed; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (globalThis as any).closed; } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -49,11 +49,18 @@ let lastThrownError: Value = undefined; // Returns true if code is consumed (no error/irrecoverable error). // Returns false if error is recoverable function evaluate(code: string): boolean { - const [result, errInfo] = core.evalContext(code); + // each evalContext is a separate function body, and we want strict mode to + // work, so we should ensure that the code starts with "use strict" + const [result, errInfo] = core.evalContext(`"use strict";\n\n${code}`); if (!errInfo) { - lastEvalResult = result; + // when a function is eval'ed with just "use strict" sometimes the result + // is "use strict" which should be discarded + lastEvalResult = + typeof result === "string" && result === "use strict" + ? undefined + : result; if (!isCloseCalled()) { - replLog(result); + replLog(lastEvalResult); } } else if (errInfo.isCompileError && isRecoverableError(errInfo.thrown)) { // Recoverable compiler error diff --git a/cli/js/runtime_main.ts b/cli/js/runtime_main.ts index 3e81fc68013313..58405f673a1111 100644 --- a/cli/js/runtime_main.ts +++ b/cli/js/runtime_main.ts @@ -30,8 +30,8 @@ import { log, immutableDefine } from "./util.ts"; // TODO: factor out `Deno` global assignment to separate function // Add internal object to Deno object. // This is not exposed as part of the Deno types. -// @ts-ignore -denoNs[internalSymbol] = internalObject; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +(denoNs as any)[internalSymbol] = internalObject; let windowIsClosing = false; @@ -71,8 +71,8 @@ export function bootstrapMainRuntime(): void { throw new Error("Worker runtime already bootstrapped"); } // Remove bootstrapping methods from global scope - // @ts-ignore - globalThis.bootstrap = undefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (globalThis as any).bootstrap = undefined; log("bootstrapMainRuntime"); hasBootstrapped = true; Object.defineProperties(globalThis, windowOrWorkerGlobalScopeMethods); diff --git a/cli/js/runtime_worker.ts b/cli/js/runtime_worker.ts index ed735fd52a8258..3f7816990dc03a 100644 --- a/cli/js/runtime_worker.ts +++ b/cli/js/runtime_worker.ts @@ -33,8 +33,8 @@ import { setSignals } from "./signals.ts"; // TODO: factor out `Deno` global assignment to separate function // Add internal object to Deno object. // This is not exposed as part of the Deno types. -// @ts-ignore -denoNs[internalSymbol] = internalObject; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +(denoNs as any)[internalSymbol] = internalObject; const encoder = new TextEncoder(); @@ -128,8 +128,8 @@ export function bootstrapWorkerRuntime( throw new Error("Worker runtime already bootstrapped"); } // Remove bootstrapping methods from global scope - // @ts-ignore - globalThis.bootstrap = undefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (globalThis as any).bootstrap = undefined; log("bootstrapWorkerRuntime"); hasBootstrapped = true; Object.defineProperties(globalThis, windowOrWorkerGlobalScopeMethods); diff --git a/cli/js/testing.ts b/cli/js/testing.ts index 09acdc23d045ce..ffe978888da780 100644 --- a/cli/js/testing.ts +++ b/cli/js/testing.ts @@ -333,12 +333,11 @@ async function runTests({ const filterFn = createFilterFn(filter, skip); const testApi = new TestApi(TEST_REGISTRY, filterFn, failFast); - // @ts-ignore const originalConsole = globalThis.console; if (disableLog) { - // @ts-ignore - globalThis.console = disabledConsole; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (globalThis as any).console = disabledConsole; } let endMsg: TestMessage["end"]; @@ -356,7 +355,6 @@ async function runTests({ } if (disableLog) { - // @ts-ignore globalThis.console = originalConsole; } diff --git a/cli/js/web/blob.ts b/cli/js/web/blob.ts index 546ea7e8200382..899c671358ea74 100644 --- a/cli/js/web/blob.ts +++ b/cli/js/web/blob.ts @@ -161,7 +161,7 @@ async function readBytes( // Ensures it does not impact garbage collection. export const blobBytesWeakMap = new WeakMap(); -export class DenoBlob implements Blob { +class DenoBlob implements Blob { [bytesSymbol]: Uint8Array; readonly size: number = 0; readonly type: string = ""; @@ -216,3 +216,11 @@ export class DenoBlob implements Blob { return readBytes(getStream(this[bytesSymbol]).getReader()); } } + +// we want the Base class name to be the name of the class. +Object.defineProperty(DenoBlob, "name", { + value: "Blob", + configurable: true, +}); + +export { DenoBlob }; diff --git a/cli/js/web/body.ts b/cli/js/web/body.ts index a1cf2038ae6a6c..672b7de5ac339b 100644 --- a/cli/js/web/body.ts +++ b/cli/js/web/body.ts @@ -2,20 +2,15 @@ import * as blob from "./blob.ts"; import * as encoding from "./text_encoding.ts"; import * as domTypes from "./dom_types.d.ts"; import { ReadableStreamImpl } from "./streams/readable_stream.ts"; +import { isReadableStreamDisturbed } from "./streams/internals.ts"; +import { getHeaderValueParams, hasHeaderValueOf } from "./util.ts"; +import { MultipartParser } from "./fetch/multipart.ts"; // only namespace imports work for now, plucking out what we need const { TextEncoder, TextDecoder } = encoding; const DenoBlob = blob.DenoBlob; -export type BodySource = - | Blob - | BufferSource - | FormData - | URLSearchParams - | ReadableStream - | string; - -function validateBodyType(owner: Body, bodySource: BodySource): boolean { +function validateBodyType(owner: Body, bodySource: BodyInit | null): boolean { if ( bodySource instanceof Int8Array || bodySource instanceof Int16Array || @@ -58,53 +53,31 @@ function concatenate(...arrays: Uint8Array[]): ArrayBuffer { return result.buffer as ArrayBuffer; } -function bufferFromStream(stream: ReadableStreamReader): Promise { - return new Promise((resolve, reject): void => { - const parts: Uint8Array[] = []; - const encoder = new TextEncoder(); - // recurse - (function pump(): void { - stream - .read() - .then(({ done, value }): void => { - if (done) { - return resolve(concatenate(...parts)); - } - - if (typeof value === "string") { - parts.push(encoder.encode(value)); - } else if (value instanceof ArrayBuffer) { - parts.push(new Uint8Array(value)); - } else if (!value) { - // noop for undefined - } else { - reject("unhandled type on stream read"); - } - - return pump(); - }) - .catch((err): void => { - reject(err); - }); - })(); - }); -} - -function getHeaderValueParams(value: string): Map { - const params = new Map(); - // Forced to do so for some Map constructor param mismatch - value - .split(";") - .slice(1) - .map((s): string[] => s.trim().split("=")) - .filter((arr): boolean => arr.length > 1) - .map(([k, v]): [string, string] => [k, v.replace(/^"([^"]*)"$/, "$1")]) - .forEach(([k, v]): Map => params.set(k, v)); - return params; -} +async function bufferFromStream( + stream: ReadableStreamReader +): Promise { + const parts: Uint8Array[] = []; + const encoder = new TextEncoder(); + + while (true) { + const { done, value } = await stream.read(); + + if (done) break; + + if (typeof value === "string") { + parts.push(encoder.encode(value)); + } else if (value instanceof ArrayBuffer) { + parts.push(new Uint8Array(value)); + } else if (value instanceof Uint8Array) { + parts.push(value); + } else if (!value) { + // noop for undefined + } else { + throw new Error("unhandled type on stream read"); + } + } -function hasHeaderValueOf(s: string, value: string): boolean { - return new RegExp(`^${value}[\t\s]*;?`).test(s); + return concatenate(...parts); } export const BodyUsedError = @@ -113,7 +86,10 @@ export const BodyUsedError = export class Body implements domTypes.Body { protected _stream: ReadableStreamImpl | null; - constructor(protected _bodySource: BodySource, readonly contentType: string) { + constructor( + protected _bodySource: BodyInit | null, + readonly contentType: string + ) { validateBodyType(this, _bodySource); this._bodySource = _bodySource; this.contentType = contentType; @@ -126,7 +102,6 @@ export class Body implements domTypes.Body { } if (this._bodySource instanceof ReadableStreamImpl) { - // @ts-ignore this._stream = this._bodySource; } if (typeof this._bodySource === "string") { @@ -142,111 +117,30 @@ export class Body implements domTypes.Body { } get bodyUsed(): boolean { - if (this.body && this.body.locked) { + if (this.body && isReadableStreamDisturbed(this.body)) { return true; } return false; } public async blob(): Promise { - return new DenoBlob([await this.arrayBuffer()]); + return new DenoBlob([await this.arrayBuffer()], { + type: this.contentType, + }); } // ref: https://fetch.spec.whatwg.org/#body-mixin public async formData(): Promise { const formData = new FormData(); - const enc = new TextEncoder(); if (hasHeaderValueOf(this.contentType, "multipart/form-data")) { const params = getHeaderValueParams(this.contentType); - if (!params.has("boundary")) { - // TypeError is required by spec - throw new TypeError("multipart/form-data must provide a boundary"); - } + // ref: https://tools.ietf.org/html/rfc2046#section-5.1 const boundary = params.get("boundary")!; - const dashBoundary = `--${boundary}`; - const delimiter = `\r\n${dashBoundary}`; - const closeDelimiter = `${delimiter}--`; + const body = new Uint8Array(await this.arrayBuffer()); + const multipartParser = new MultipartParser(body, boundary); - const body = await this.text(); - let bodyParts: string[]; - const bodyEpilogueSplit = body.split(closeDelimiter); - if (bodyEpilogueSplit.length < 2) { - bodyParts = []; - } else { - // discard epilogue - const bodyEpilogueTrimmed = bodyEpilogueSplit[0]; - // first boundary treated special due to optional prefixed \r\n - const firstBoundaryIndex = bodyEpilogueTrimmed.indexOf(dashBoundary); - if (firstBoundaryIndex < 0) { - throw new TypeError("Invalid boundary"); - } - const bodyPreambleTrimmed = bodyEpilogueTrimmed - .slice(firstBoundaryIndex + dashBoundary.length) - .replace(/^[\s\r\n\t]+/, ""); // remove transport-padding CRLF - // trimStart might not be available - // Be careful! body-part allows trailing \r\n! - // (as long as it is not part of `delimiter`) - bodyParts = bodyPreambleTrimmed - .split(delimiter) - .map((s): string => s.replace(/^[\s\r\n\t]+/, "")); - // TODO: LWSP definition is actually trickier, - // but should be fine in our case since without headers - // we should just discard the part - } - for (const bodyPart of bodyParts) { - const headers = new Headers(); - const headerOctetSeperatorIndex = bodyPart.indexOf("\r\n\r\n"); - if (headerOctetSeperatorIndex < 0) { - continue; // Skip unknown part - } - const headerText = bodyPart.slice(0, headerOctetSeperatorIndex); - const octets = bodyPart.slice(headerOctetSeperatorIndex + 4); - - // TODO: use textproto.readMIMEHeader from deno_std - const rawHeaders = headerText.split("\r\n"); - for (const rawHeader of rawHeaders) { - const sepIndex = rawHeader.indexOf(":"); - if (sepIndex < 0) { - continue; // Skip this header - } - const key = rawHeader.slice(0, sepIndex); - const value = rawHeader.slice(sepIndex + 1); - headers.set(key, value); - } - if (!headers.has("content-disposition")) { - continue; // Skip unknown part - } - // Content-Transfer-Encoding Deprecated - const contentDisposition = headers.get("content-disposition")!; - const partContentType = headers.get("content-type") || "text/plain"; - // TODO: custom charset encoding (needs TextEncoder support) - // const contentTypeCharset = - // getHeaderValueParams(partContentType).get("charset") || ""; - if (!hasHeaderValueOf(contentDisposition, "form-data")) { - continue; // Skip, might not be form-data - } - const dispositionParams = getHeaderValueParams(contentDisposition); - if (!dispositionParams.has("name")) { - continue; // Skip, unknown name - } - const dispositionName = dispositionParams.get("name")!; - if (dispositionParams.has("filename")) { - const filename = dispositionParams.get("filename")!; - const blob = new DenoBlob([enc.encode(octets)], { - type: partContentType, - }); - // TODO: based on spec - // https://xhr.spec.whatwg.org/#dom-formdata-append - // https://xhr.spec.whatwg.org/#create-an-entry - // Currently it does not mention how I could pass content-type - // to the internally created file object... - formData.append(dispositionName, blob, filename); - } else { - formData.append(dispositionName, octets); - } - } - return formData; + return multipartParser.parse(); } else if ( hasHeaderValueOf(this.contentType, "application/x-www-form-urlencoded") ) { @@ -314,7 +208,6 @@ export class Body implements domTypes.Body { enc.encode(this._bodySource).buffer as ArrayBuffer ); } else if (this._bodySource instanceof ReadableStreamImpl) { - // @ts-ignore return bufferFromStream(this._bodySource.getReader()); } else if (this._bodySource instanceof FormData) { const enc = new TextEncoder(); diff --git a/cli/js/web/console.ts b/cli/js/web/console.ts index 7a3f9241e4435f..9f166b373f6bf3 100644 --- a/cli/js/web/console.ts +++ b/cli/js/web/console.ts @@ -124,23 +124,19 @@ function createIterableString( const iPrefix = `${config.displayName ? config.displayName + " " : ""}`; + const initIndentation = `\n${DEFAULT_INDENT.repeat(level + 1)}`; + const entryIndentation = `,\n${DEFAULT_INDENT.repeat(level + 1)}`; + const closingIndentation = `\n${DEFAULT_INDENT.repeat(level)}`; + let iContent: string; if (config.group && entries.length > MIN_GROUP_LENGTH) { const groups = groupEntries(entries, level, value); - const initIndentation = `\n${DEFAULT_INDENT.repeat(level + 1)}`; - const entryIndentation = `,\n${DEFAULT_INDENT.repeat(level + 1)}`; - const closingIndentation = `\n${DEFAULT_INDENT.repeat(level)}`; - iContent = `${initIndentation}${groups.join( entryIndentation )}${closingIndentation}`; } else { iContent = entries.length === 0 ? "" : ` ${entries.join(", ")} `; if (stripColor(iContent).length > LINE_BREAKING_LENGTH) { - const initIndentation = `\n${DEFAULT_INDENT.repeat(level + 1)}`; - const entryIndentation = `,\n${DEFAULT_INDENT.repeat(level + 1)}`; - const closingIndentation = `\n`; - iContent = `${initIndentation}${entries.join( entryIndentation )}${closingIndentation}`; @@ -220,14 +216,18 @@ function groupEntries( lineMaxLength += separatorSpace; maxLineLength[i] = lineMaxLength; } - let order = "padStart"; + let order: "padStart" | "padEnd" = "padStart"; if (value !== undefined) { for (let i = 0; i < entries.length; i++) { - //@ts-ignore - if (typeof value[i] !== "number" && typeof value[i] !== "bigint") { + /* eslint-disable @typescript-eslint/no-explicit-any */ + if ( + typeof (value as any)[i] !== "number" && + typeof (value as any)[i] !== "bigint" + ) { order = "padEnd"; break; } + /* eslint-enable */ } } // Each iteration creates a single line of grouped entries. @@ -239,7 +239,6 @@ function groupEntries( for (; j < max - 1; j++) { // In future, colors should be taken here into the account const padding = maxLineLength[j - i]; - //@ts-ignore str += `${entries[j]}, `[order](padding, " "); } if (order === "padStart") { @@ -343,7 +342,7 @@ function createArrayString( const ending = emptyItems > 1 ? "s" : ""; return dim(`<${emptyItems} empty item${ending}>`); } else { - return stringifyWithQuotes(val, ctx, level + 1, maxLevel); + return stringifyWithQuotes(val, ctx, level, maxLevel); } }, group: true, @@ -412,8 +411,8 @@ function createMapString( }, group: false, }; - //@ts-ignore - return createIterableString(value, ctx, level, maxLevel, printConfig); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return createIterableString(value as any, ctx, level, maxLevel, printConfig); } function createWeakSetString(): string { @@ -481,7 +480,7 @@ function createPromiseString( // TODO: Proxy function createRawObjectString( - value: { [key: string]: unknown }, + value: Record, ctx: ConsoleContext, level: number, maxLevel: number @@ -494,8 +493,9 @@ function createRawObjectString( let baseString = ""; let shouldShowDisplayName = false; - // @ts-ignore - let displayName = value[Symbol.toStringTag]; + let displayName = (value as { [Symbol.toStringTag]: string })[ + Symbol.toStringTag + ]; if (!displayName) { displayName = getClassInstanceName(value); } @@ -515,8 +515,8 @@ function createRawObjectString( for (const key of symbolKeys) { entries.push( `${key.toString()}: ${stringifyWithQuotes( - // @ts-ignore - value[key], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + value[key as any], ctx, level + 1, maxLevel @@ -842,27 +842,33 @@ export class Console { resultData = data!; } + let hasPrimitives = false; Object.keys(resultData).forEach((k, idx): void => { const value: unknown = resultData[k]!; - - if (value !== null && typeof value === "object") { - Object.entries(value as { [key: string]: unknown }).forEach( - ([k, v]): void => { - if (properties && !properties.includes(k)) { - return; + const primitive = + value === null || + (typeof value !== "function" && typeof value !== "object"); + if (properties === undefined && primitive) { + hasPrimitives = true; + values.push(stringifyValue(value)); + } else { + const valueObj = (value as { [key: string]: unknown }) || {}; + const keys = properties || Object.keys(valueObj); + for (const k of keys) { + if (primitive || !valueObj.hasOwnProperty(k)) { + if (objectValues[k]) { + // fill with blanks for idx to avoid misplacing from later values + objectValues[k].push(""); } - + } else { if (objectValues[k]) { - objectValues[k].push(stringifyValue(v)); + objectValues[k].push(stringifyValue(valueObj[k])); } else { - objectValues[k] = createColumn(v, idx); + objectValues[k] = createColumn(valueObj[k], idx); } } - ); - + } values.push(""); - } else { - values.push(stringifyValue(value)); } indexKeys.push(k); @@ -872,10 +878,7 @@ export class Console { const bodyValues = Object.values(objectValues); const header = [ indexKey, - ...(properties || [ - ...headerKeys, - !isMap && values.length > 0 && valuesKey, - ]), + ...(properties || [...headerKeys, !isMap && hasPrimitives && valuesKey]), ].filter(Boolean) as string[]; const body = [indexKeys, ...bodyValues, values]; @@ -949,7 +952,6 @@ export class Console { name: "Trace", message, }; - // @ts-ignore Error.captureStackTrace(err, this.trace); this.error((err as Error).stack); }; diff --git a/cli/js/web/dom_types.d.ts b/cli/js/web/dom_types.d.ts index b5b172ccde68c2..5d35c91875147c 100644 --- a/cli/js/web/dom_types.d.ts +++ b/cli/js/web/dom_types.d.ts @@ -305,7 +305,6 @@ export interface Response extends Body { readonly redirected: boolean; readonly status: number; readonly statusText: string; - readonly trailer: Promise; readonly type: ResponseType; readonly url: string; clone(): Response; diff --git a/cli/js/web/fetch.ts b/cli/js/web/fetch.ts index 38ca03aca7be5a..045d1afcd22b59 100644 --- a/cli/js/web/fetch.ts +++ b/cli/js/web/fetch.ts @@ -1,328 +1,79 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assert, createResolvable, notImplemented } from "../util.ts"; +import { notImplemented } from "../util.ts"; import { isTypedArray } from "./util.ts"; import * as domTypes from "./dom_types.d.ts"; -import { TextDecoder, TextEncoder } from "./text_encoding.ts"; +import { TextEncoder } from "./text_encoding.ts"; import { DenoBlob, bytesSymbol as blobBytesSymbol } from "./blob.ts"; -import * as io from "../io.ts"; import { read } from "../ops/io.ts"; import { close } from "../ops/resources.ts"; -import { Buffer } from "../buffer.ts"; import { fetch as opFetch, FetchResponse } from "../ops/fetch.ts"; -import { DomFileImpl } from "./dom_file.ts"; - -function getHeaderValueParams(value: string): Map { - const params = new Map(); - // Forced to do so for some Map constructor param mismatch - value - .split(";") - .slice(1) - .map((s): string[] => s.trim().split("=")) - .filter((arr): boolean => arr.length > 1) - .map(([k, v]): [string, string] => [k, v.replace(/^"([^"]*)"$/, "$1")]) - .forEach(([k, v]): Map => params.set(k, v)); - return params; -} - -function hasHeaderValueOf(s: string, value: string): boolean { - return new RegExp(`^${value}[\t\s]*;?`).test(s); -} - -class Body - implements domTypes.Body, ReadableStream, io.Reader, io.Closer { - #bodyUsed = false; - #bodyPromise: Promise | null = null; - #data: ArrayBuffer | null = null; - #rid: number; - readonly locked: boolean = false; // TODO - readonly body: ReadableStream; - - constructor(rid: number, readonly contentType: string) { - this.#rid = rid; - this.body = this; - } +import * as Body from "./body.ts"; +import { getHeaderValueParams } from "./util.ts"; +import { ReadableStreamImpl } from "./streams/readable_stream.ts"; +import { MultipartBuilder } from "./fetch/multipart.ts"; - #bodyBuffer = async (): Promise => { - assert(this.#bodyPromise == null); - const buf = new Buffer(); - try { - const nread = await buf.readFrom(this); - const ui8 = buf.bytes(); - assert(ui8.byteLength === nread); - this.#data = ui8.buffer.slice( - ui8.byteOffset, - ui8.byteOffset + nread - ) as ArrayBuffer; - assert(this.#data.byteLength === nread); - } finally { - this.close(); - } +const NULL_BODY_STATUS = [101, 204, 205, 304]; +const REDIRECT_STATUS = [301, 302, 303, 307, 308]; - return this.#data; - }; +const responseData = new WeakMap(); +export class Response extends Body.Body implements domTypes.Response { + readonly type: ResponseType; + readonly redirected: boolean; + readonly url: string; + readonly status: number; + readonly statusText: string; + headers: Headers; - // eslint-disable-next-line require-await - async arrayBuffer(): Promise { - // If we've already bufferred the response, just return it. - if (this.#data != null) { - return this.#data; - } + constructor(body: BodyInit | null = null, init?: domTypes.ResponseInit) { + init = init ?? {}; - // If there is no _bodyPromise yet, start it. - if (this.#bodyPromise == null) { - this.#bodyPromise = this.#bodyBuffer(); + if (typeof init !== "object") { + throw new TypeError(`'init' is not an object`); } - return this.#bodyPromise; - } + const extraInit = responseData.get(init) || {}; + let { type = "default", url = "" } = extraInit; - async blob(): Promise { - const arrayBuffer = await this.arrayBuffer(); - return new DenoBlob([arrayBuffer], { - type: this.contentType, - }); - } + let status = (Number(init.status) || 0) ?? 200; + let statusText = init.statusText ?? ""; + let headers = + init.headers instanceof Headers + ? init.headers + : new Headers(init.headers); - // ref: https://fetch.spec.whatwg.org/#body-mixin - async formData(): Promise { - const formData = new FormData(); - const enc = new TextEncoder(); - if (hasHeaderValueOf(this.contentType, "multipart/form-data")) { - const params = getHeaderValueParams(this.contentType); - if (!params.has("boundary")) { - // TypeError is required by spec - throw new TypeError("multipart/form-data must provide a boundary"); - } - // ref: https://tools.ietf.org/html/rfc2046#section-5.1 - const boundary = params.get("boundary")!; - const dashBoundary = `--${boundary}`; - const delimiter = `\r\n${dashBoundary}`; - const closeDelimiter = `${delimiter}--`; - - const body = await this.text(); - let bodyParts: string[]; - const bodyEpilogueSplit = body.split(closeDelimiter); - if (bodyEpilogueSplit.length < 2) { - bodyParts = []; - } else { - // discard epilogue - const bodyEpilogueTrimmed = bodyEpilogueSplit[0]; - // first boundary treated special due to optional prefixed \r\n - const firstBoundaryIndex = bodyEpilogueTrimmed.indexOf(dashBoundary); - if (firstBoundaryIndex < 0) { - throw new TypeError("Invalid boundary"); - } - const bodyPreambleTrimmed = bodyEpilogueTrimmed - .slice(firstBoundaryIndex + dashBoundary.length) - .replace(/^[\s\r\n\t]+/, ""); // remove transport-padding CRLF - // trimStart might not be available - // Be careful! body-part allows trailing \r\n! - // (as long as it is not part of `delimiter`) - bodyParts = bodyPreambleTrimmed - .split(delimiter) - .map((s): string => s.replace(/^[\s\r\n\t]+/, "")); - // TODO: LWSP definition is actually trickier, - // but should be fine in our case since without headers - // we should just discard the part - } - for (const bodyPart of bodyParts) { - const headers = new Headers(); - const headerOctetSeperatorIndex = bodyPart.indexOf("\r\n\r\n"); - if (headerOctetSeperatorIndex < 0) { - continue; // Skip unknown part - } - const headerText = bodyPart.slice(0, headerOctetSeperatorIndex); - const octets = bodyPart.slice(headerOctetSeperatorIndex + 4); - - // TODO: use textproto.readMIMEHeader from deno_std - const rawHeaders = headerText.split("\r\n"); - for (const rawHeader of rawHeaders) { - const sepIndex = rawHeader.indexOf(":"); - if (sepIndex < 0) { - continue; // Skip this header - } - const key = rawHeader.slice(0, sepIndex); - const value = rawHeader.slice(sepIndex + 1); - headers.set(key, value); - } - if (!headers.has("content-disposition")) { - continue; // Skip unknown part - } - // Content-Transfer-Encoding Deprecated - const contentDisposition = headers.get("content-disposition")!; - const partContentType = headers.get("content-type") || "text/plain"; - // TODO: custom charset encoding (needs TextEncoder support) - // const contentTypeCharset = - // getHeaderValueParams(partContentType).get("charset") || ""; - if (!hasHeaderValueOf(contentDisposition, "form-data")) { - continue; // Skip, might not be form-data - } - const dispositionParams = getHeaderValueParams(contentDisposition); - if (!dispositionParams.has("name")) { - continue; // Skip, unknown name - } - const dispositionName = dispositionParams.get("name")!; - if (dispositionParams.has("filename")) { - const filename = dispositionParams.get("filename")!; - const blob = new DenoBlob([enc.encode(octets)], { - type: partContentType, - }); - // TODO: based on spec - // https://xhr.spec.whatwg.org/#dom-formdata-append - // https://xhr.spec.whatwg.org/#create-an-entry - // Currently it does not mention how I could pass content-type - // to the internally created file object... - formData.append(dispositionName, blob, filename); - } else { - formData.append(dispositionName, octets); - } - } - return formData; - } else if ( - hasHeaderValueOf(this.contentType, "application/x-www-form-urlencoded") - ) { - // From https://github.com/github/fetch/blob/master/fetch.js - // Copyright (c) 2014-2016 GitHub, Inc. MIT License - const body = await this.text(); - try { - body - .trim() - .split("&") - .forEach((bytes): void => { - if (bytes) { - const split = bytes.split("="); - const name = split.shift()!.replace(/\+/g, " "); - const value = split.join("=").replace(/\+/g, " "); - formData.append( - decodeURIComponent(name), - decodeURIComponent(value) - ); - } - }); - } catch (e) { - throw new TypeError("Invalid form urlencoded format"); - } - return formData; - } else { - throw new TypeError("Invalid form data"); + if (init.status && (status < 200 || status > 599)) { + throw new RangeError( + `The status provided (${init.status}) is outside the range [200, 599]` + ); } - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async json(): Promise { - const text = await this.text(); - return JSON.parse(text); - } - - async text(): Promise { - const ab = await this.arrayBuffer(); - const decoder = new TextDecoder("utf-8"); - return decoder.decode(ab); - } - - read(p: Uint8Array): Promise { - this.#bodyUsed = true; - return read(this.#rid, p); - } - - close(): Promise { - close(this.#rid); - return Promise.resolve(); - } - - cancel(): Promise { - return notImplemented(); - } - - getIterator(_options?: { - preventCancel?: boolean; - }): AsyncIterableIterator { - return notImplemented(); - } - - getReader(): ReadableStreamDefaultReader { - return notImplemented(); - } - - tee(): [ReadableStream, ReadableStream] { - return notImplemented(); - } - - [Symbol.asyncIterator](): AsyncIterableIterator { - return io.iter(this); - } - - get bodyUsed(): boolean { - return this.#bodyUsed; - } - - pipeThrough( - _: { - writable: WritableStream; - readable: ReadableStream; - }, - _options?: PipeOptions - ): ReadableStream { - return notImplemented(); - } - pipeTo( - _dest: WritableStream, - _options?: PipeOptions - ): Promise { - return notImplemented(); - } -} - -export class Response implements domTypes.Response { - readonly type: ResponseType; - readonly redirected: boolean; - headers: Headers; - readonly trailer: Promise; - readonly body: Body | null; - - constructor( - readonly url: string, - readonly status: number, - readonly statusText: string, - headersList: Array<[string, string]>, - rid: number, - redirected_: boolean, - readonly type_: null | ResponseType = "default", - body_: null | Body = null - ) { - this.trailer = createResolvable(); - this.headers = new Headers(headersList); - const contentType = this.headers.get("content-type") || ""; - - if (body_ == null) { - this.body = new Body(rid, contentType); - } else { - this.body = body_; + // null body status + if (body && NULL_BODY_STATUS.includes(status)) { + throw new TypeError("Response with null body status cannot have body"); } - if (type_ == null) { - this.type = "default"; + if (!type) { + type = "default"; } else { - this.type = type_; - if (type_ == "error") { + type = type; + if (type == "error") { // spec: https://fetch.spec.whatwg.org/#concept-network-error - this.status = 0; - this.statusText = ""; - this.headers = new Headers(); - this.body = null; + status = 0; + statusText = ""; + headers = new Headers(); + body = null; /* spec for other Response types: https://fetch.spec.whatwg.org/#concept-filtered-response-basic Please note that type "basic" is not the same thing as "default".*/ - } else if (type_ == "basic") { - for (const h of this.headers) { + } else if (type == "basic") { + for (const h of headers) { /* Forbidden Response-Header Names: https://fetch.spec.whatwg.org/#forbidden-response-header-name */ if (["set-cookie", "set-cookie2"].includes(h[0].toLowerCase())) { - this.headers.delete(h[0]); + headers.delete(h[0]); } } - } else if (type_ == "cors") { + } else if (type == "cors") { /* CORS-safelisted Response-Header Names: https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name */ const allowedHeaders = [ @@ -334,7 +85,7 @@ export class Response implements domTypes.Response { "Last-Modified", "Pragma", ].map((c: string) => c.toLowerCase()); - for (const h of this.headers) { + for (const h of headers) { /* Technically this is still not standards compliant because we are supposed to allow headers allowed in the 'Access-Control-Expose-Headers' header in the 'internal response' @@ -344,87 +95,39 @@ export class Response implements domTypes.Response { TODO(serverhiccups): change how internal responses are handled so we can do this properly. */ if (!allowedHeaders.includes(h[0].toLowerCase())) { - this.headers.delete(h[0]); + headers.delete(h[0]); } } /* TODO(serverhiccups): Once I fix the 'internal response' thing, these actually need to treat the internal response differently */ - } else if (type_ == "opaque" || type_ == "opaqueredirect") { - this.url = ""; - this.status = 0; - this.statusText = ""; - this.headers = new Headers(); - this.body = null; + } else if (type == "opaque" || type == "opaqueredirect") { + url = ""; + status = 0; + statusText = ""; + headers = new Headers(); + body = null; } } - this.redirected = redirected_; - } + const contentType = headers.get("content-type") || ""; - #bodyViewable = (): boolean => { - if ( - this.type == "error" || - this.type == "opaque" || - this.type == "opaqueredirect" || - this.body == undefined - ) { - return true; - } - return false; - }; + super(body, contentType); - arrayBuffer(): Promise { - /* You have to do the null check here and not in the function because - * otherwise TS complains about this.body potentially being null */ - if (this.#bodyViewable() || this.body == null) { - return Promise.reject(new Error("Response body is null")); - } - return this.body.arrayBuffer(); - } - - blob(): Promise { - if (this.#bodyViewable() || this.body == null) { - return Promise.reject(new Error("Response body is null")); - } - return this.body.blob(); - } - - formData(): Promise { - if (this.#bodyViewable() || this.body == null) { - return Promise.reject(new Error("Response body is null")); - } - return this.body.formData(); - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - json(): Promise { - if (this.#bodyViewable() || this.body == null) { - return Promise.reject(new Error("Response body is null")); - } - return this.body.json(); - } - - text(): Promise { - if (this.#bodyViewable() || this.body == null) { - return Promise.reject(new Error("Response body is null")); - } - return this.body.text(); + this.url = url; + this.statusText = statusText; + this.status = extraInit.status || status; + this.headers = headers; + this.redirected = extraInit.redirected; + this.type = type; } get ok(): boolean { return 200 <= this.status && this.status < 300; } - get bodyUsed(): boolean { - if (this.body === null) return false; - return this.body.bodyUsed; - } - - clone(): domTypes.Response { + public clone(): domTypes.Response { if (this.bodyUsed) { - throw new TypeError( - "Failed to execute 'clone' on 'Response': Response body is already used" - ); + throw TypeError(Body.BodyUsedError); } const iterators = this.headers.entries(); @@ -433,16 +136,20 @@ export class Response implements domTypes.Response { headersList.push(header); } - return new Response( - this.url, - this.status, - this.statusText, - headersList, - -1, - this.redirected, - this.type, - this.body - ); + let resBody = this._bodySource; + + if (this._bodySource instanceof ReadableStreamImpl) { + const tees = this._bodySource.tee(); + this._stream = this._bodySource = tees[0]; + resBody = tees[1]; + } + + const cloned = new Response(resBody, { + status: this.status, + statusText: this.statusText, + headers: new Headers(headersList), + }); + return cloned; } static redirect(url: URL | string, status: number): domTypes.Response { @@ -451,16 +158,11 @@ export class Response implements domTypes.Response { "The redirection status must be one of 301, 302, 303, 307 and 308." ); } - return new Response( - "", + return new Response(null, { status, - "", - [["Location", typeof url === "string" ? url : url.toString()]], - -1, - false, - "default", - null - ); + statusText: "", + headers: [["Location", typeof url === "string" ? url : url.toString()]], + }); } } @@ -485,7 +187,7 @@ function sendFetchReq( } export async function fetch( - input: domTypes.Request | URL | string, + input: (domTypes.Request & { _bodySource?: unknown }) | URL | string, init?: domTypes.RequestInit ): Promise { let url: string; @@ -521,6 +223,8 @@ export async function fetch( contentType = "text/plain;charset=UTF-8"; } else if (isTypedArray(init.body)) { body = init.body; + } else if (init.body instanceof ArrayBuffer) { + body = new Uint8Array(init.body); } else if (init.body instanceof URLSearchParams) { body = new TextEncoder().encode(init.body.toString()); contentType = "application/x-www-form-urlencoded;charset=UTF-8"; @@ -528,45 +232,14 @@ export async function fetch( body = init.body[blobBytesSymbol]; contentType = init.body.type; } else if (init.body instanceof FormData) { - let boundary = ""; + let boundary; if (headers.has("content-type")) { const params = getHeaderValueParams("content-type"); - if (params.has("boundary")) { - boundary = params.get("boundary")!; - } - } - if (!boundary) { - boundary = - "----------" + - Array.from(Array(32)) - .map(() => Math.random().toString(36)[2] || 0) - .join(""); + boundary = params.get("boundary")!; } - - let payload = ""; - for (const [fieldName, fieldValue] of init.body.entries()) { - let part = `\r\n--${boundary}\r\n`; - part += `Content-Disposition: form-data; name=\"${fieldName}\"`; - if (fieldValue instanceof DomFileImpl) { - part += `; filename=\"${fieldValue.name}\"`; - } - part += "\r\n"; - if (fieldValue instanceof DomFileImpl) { - part += `Content-Type: ${ - fieldValue.type || "application/octet-stream" - }\r\n`; - } - part += "\r\n"; - if (fieldValue instanceof DomFileImpl) { - part += new TextDecoder().decode(fieldValue[blobBytesSymbol]); - } else { - part += fieldValue; - } - payload += part; - } - payload += `\r\n--${boundary}--`; - body = new TextEncoder().encode(payload); - contentType = "multipart/form-data; boundary=" + boundary; + const multipartBuilder = new MultipartBuilder(init.body, boundary); + body = multipartBuilder.getBody(); + contentType = multipartBuilder.getContentType(); } else { // TODO: ReadableStream notImplemented(); @@ -581,35 +254,83 @@ export async function fetch( method = input.method; headers = input.headers; - //@ts-ignore if (input._bodySource) { body = new DataView(await input.arrayBuffer()); } } + let responseBody; + let responseInit: ResponseInit = {}; while (remRedirectCount) { const fetchResponse = await sendFetchReq(url, method, headers, body); - const response = new Response( - url, - fetchResponse.status, - fetchResponse.statusText, - fetchResponse.headers, - fetchResponse.bodyRid, - redirected - ); - if ([301, 302, 303, 307, 308].includes(response.status)) { + if ( + NULL_BODY_STATUS.includes(fetchResponse.status) || + REDIRECT_STATUS.includes(fetchResponse.status) + ) { // We won't use body of received response, so close it now // otherwise it will be kept in resource table. close(fetchResponse.bodyRid); + responseBody = null; + } else { + responseBody = new ReadableStreamImpl({ + async pull(controller: ReadableStreamDefaultController): Promise { + try { + const b = new Uint8Array(1024 * 32); + const result = await read(fetchResponse.bodyRid, b); + if (result === null) { + controller.close(); + return close(fetchResponse.bodyRid); + } + + controller.enqueue(b.subarray(0, result)); + } catch (e) { + controller.error(e); + controller.close(); + close(fetchResponse.bodyRid); + } + }, + cancel(): void { + // When reader.cancel() is called + close(fetchResponse.bodyRid); + }, + }); + } + + responseInit = { + status: 200, + statusText: fetchResponse.statusText, + headers: fetchResponse.headers, + }; + + responseData.set(responseInit, { + redirected, + rid: fetchResponse.bodyRid, + status: fetchResponse.status, + url, + }); + + const response = new Response(responseBody, responseInit); + + if (REDIRECT_STATUS.includes(fetchResponse.status)) { // We're in a redirect status switch ((init && init.redirect) || "follow") { case "error": - /* I suspect that deno will probably crash if you try to use that - rid, which suggests to me that Response needs to be refactored */ - return new Response("", 0, "", [], -1, false, "error", null); + responseInit = {}; + responseData.set(responseInit, { + type: "error", + redirected: false, + url: "", + }); + return new Response(null, responseInit); case "manual": - return new Response("", 0, "", [], -1, false, "opaqueredirect", null); + responseInit = {}; + responseData.set(responseInit, { + type: "opaqueredirect", + redirected: false, + url: "", + }); + return new Response(null, responseInit); case "follow": default: let redirectUrl = response.headers.get("Location"); @@ -634,6 +355,12 @@ export async function fetch( return response; } } - // Return a network error due to too many redirections - throw notImplemented(); + + responseData.set(responseInit, { + type: "error", + redirected: false, + url: "", + }); + + return new Response(null, responseInit); } diff --git a/cli/js/web/fetch/multipart.ts b/cli/js/web/fetch/multipart.ts new file mode 100644 index 00000000000000..654d4a0ea9ced2 --- /dev/null +++ b/cli/js/web/fetch/multipart.ts @@ -0,0 +1,201 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +import { Buffer } from "../../buffer.ts"; +import { bytesSymbol } from "../blob.ts"; +import { DomFileImpl } from "../dom_file.ts"; +import { DenoBlob } from "../blob.ts"; +import { TextEncoder, TextDecoder } from "../text_encoding.ts"; +import { getHeaderValueParams } from "../util.ts"; + +const decoder = new TextDecoder(); +const encoder = new TextEncoder(); +const CR = "\r".charCodeAt(0); +const LF = "\n".charCodeAt(0); + +interface MultipartHeaders { + headers: Headers; + disposition: Map; +} + +export class MultipartBuilder { + readonly boundary: string; + readonly formData: FormData; + readonly writer: Buffer; + constructor(formData: FormData, boundary?: string) { + this.boundary = boundary ?? this.#createBoundary(); + this.formData = formData; + this.writer = new Buffer(); + } + + getContentType(): string { + return `multipart/form-data; boundary=${this.boundary}`; + } + + getBody(): Uint8Array { + for (const [fieldName, fieldValue] of this.formData.entries()) { + if (fieldValue instanceof DomFileImpl) { + this.#writeFile(fieldName, fieldValue); + } else this.#writeField(fieldName, fieldValue as string); + } + + this.writer.writeSync(encoder.encode(`\r\n--${this.boundary}--`)); + + return this.writer.bytes(); + } + + #createBoundary = (): string => { + return ( + "----------" + + Array.from(Array(32)) + .map(() => Math.random().toString(36)[2] || 0) + .join("") + ); + }; + + #writeHeaders = (headers: string[][]): void => { + let buf = this.writer.empty() ? "" : "\r\n"; + + buf += `--${this.boundary}\r\n`; + for (const [key, value] of headers) { + buf += `${key}: ${value}\r\n`; + } + buf += `\r\n`; + + this.writer.write(encoder.encode(buf)); + }; + + #writeFileHeaders = ( + field: string, + filename: string, + type?: string + ): void => { + const headers = [ + [ + "Content-Disposition", + `form-data; name="${field}"; filename="${filename}"`, + ], + ["Content-Type", type || "application/octet-stream"], + ]; + return this.#writeHeaders(headers); + }; + + #writeFieldHeaders = (field: string): void => { + const headers = [["Content-Disposition", `form-data; name="${field}"`]]; + return this.#writeHeaders(headers); + }; + + #writeField = (field: string, value: string): void => { + this.#writeFieldHeaders(field); + this.writer.writeSync(encoder.encode(value)); + }; + + #writeFile = (field: string, value: DomFileImpl): void => { + this.#writeFileHeaders(field, value.name, value.type); + this.writer.writeSync(value[bytesSymbol]); + }; +} + +export class MultipartParser { + readonly boundary: string; + readonly boundaryChars: Uint8Array; + readonly body: Uint8Array; + constructor(body: Uint8Array, boundary: string) { + if (!boundary) { + throw new TypeError("multipart/form-data must provide a boundary"); + } + + this.boundary = `--${boundary}`; + this.body = body; + this.boundaryChars = encoder.encode(this.boundary); + } + + #parseHeaders = (headersText: string): MultipartHeaders => { + const headers = new Headers(); + const rawHeaders = headersText.split("\r\n"); + for (const rawHeader of rawHeaders) { + const sepIndex = rawHeader.indexOf(":"); + if (sepIndex < 0) { + continue; // Skip this header + } + const key = rawHeader.slice(0, sepIndex); + const value = rawHeader.slice(sepIndex + 1); + headers.set(key, value); + } + + return { + headers, + disposition: getHeaderValueParams( + headers.get("Content-Disposition") ?? "" + ), + }; + }; + + parse(): FormData { + const formData = new FormData(); + let headerText = ""; + let boundaryIndex = 0; + let state = 0; + let fileStart = 0; + + for (let i = 0; i < this.body.length; i++) { + const byte = this.body[i]; + const prevByte = this.body[i - 1]; + const isNewLine = byte === LF && prevByte === CR; + + if (state === 1 || state === 2 || state == 3) { + headerText += String.fromCharCode(byte); + } + if (state === 0 && isNewLine) { + state = 1; + } else if (state === 1 && isNewLine) { + state = 2; + const headersDone = this.body[i + 1] === CR && this.body[i + 2] === LF; + + if (headersDone) { + state = 3; + } + } else if (state === 2 && isNewLine) { + state = 3; + } else if (state === 3 && isNewLine) { + state = 4; + fileStart = i + 1; + } else if (state === 4) { + if (this.boundaryChars[boundaryIndex] !== byte) { + boundaryIndex = 0; + } else { + boundaryIndex++; + } + + if (boundaryIndex >= this.boundary.length) { + const { headers, disposition } = this.#parseHeaders(headerText); + const content = this.body.subarray(fileStart, i - boundaryIndex - 1); + // https://fetch.spec.whatwg.org/#ref-for-dom-body-formdata + const filename = disposition.get("filename"); + const name = disposition.get("name"); + + state = 5; + // Reset + boundaryIndex = 0; + headerText = ""; + + if (!name) { + continue; // Skip, unknown name + } + + if (filename) { + const blob = new DenoBlob([content], { + type: headers.get("Content-Type") || "application/octet-stream", + }); + formData.append(name, blob, filename); + } else { + formData.append(name, decoder.decode(content)); + } + } + } else if (state === 5 && isNewLine) { + state = 1; + } + } + + return formData; + } +} diff --git a/cli/js/web/form_data.ts b/cli/js/web/form_data.ts index 5fab02553f71d0..155f40771aef1b 100644 --- a/cli/js/web/form_data.ts +++ b/cli/js/web/form_data.ts @@ -22,7 +22,7 @@ class FormDataBase { if (value instanceof domFile.DomFileImpl) { this[dataSymbol].push([name, value]); } else if (value instanceof blob.DenoBlob) { - const dfile = new domFile.DomFileImpl([value], filename || name, { + const dfile = new domFile.DomFileImpl([value], filename || "blob", { type: value.type, }); this[dataSymbol].push([name, dfile]); @@ -96,7 +96,7 @@ class FormDataBase { if (value instanceof domFile.DomFileImpl) { this[dataSymbol][i][1] = value; } else if (value instanceof blob.DenoBlob) { - const dfile = new domFile.DomFileImpl([value], filename || name, { + const dfile = new domFile.DomFileImpl([value], filename || "blob", { type: value.type, }); this[dataSymbol][i][1] = dfile; @@ -117,7 +117,7 @@ class FormDataBase { if (value instanceof domFile.DomFileImpl) { this[dataSymbol].push([name, value]); } else if (value instanceof blob.DenoBlob) { - const dfile = new domFile.DomFileImpl([value], filename || name, { + const dfile = new domFile.DomFileImpl([value], filename || "blob", { type: value.type, }); this[dataSymbol].push([name, dfile]); @@ -137,3 +137,8 @@ export class FormDataImpl extends DomIterableMixin< FormDataEntryValue, typeof FormDataBase >(FormDataBase, dataSymbol) {} + +Object.defineProperty(FormDataImpl, "name", { + value: "FormData", + configurable: true, +}); diff --git a/cli/js/web/headers.ts b/cli/js/web/headers.ts index d803f69f3026c8..9e0a70f0d4471b 100644 --- a/cli/js/web/headers.ts +++ b/cli/js/web/headers.ts @@ -256,3 +256,8 @@ export class HeadersImpl extends DomIterableMixin< string, typeof HeadersBase >(HeadersBase, headersData) {} + +Object.defineProperty(HeadersImpl, "name", { + value: "Headers", + configurable: true, +}); diff --git a/cli/js/web/request.ts b/cli/js/web/request.ts index 8fe93babe83479..286aaff56eb0fa 100644 --- a/cli/js/web/request.ts +++ b/cli/js/web/request.ts @@ -39,7 +39,7 @@ export class Request extends body.Body implements domTypes.Request { init = {}; } - let b: body.BodySource; + let b: BodyInit; // prefer body from init if (init.body) { diff --git a/cli/js/web/streams/internals.ts b/cli/js/web/streams/internals.ts index ff8fc25feef6f9..d0d35a1c37e8e2 100644 --- a/cli/js/web/streams/internals.ts +++ b/cli/js/web/streams/internals.ts @@ -367,6 +367,11 @@ export function isReadableStreamLocked(stream: ReadableStreamImpl): boolean { return stream[sym.reader] ? true : false; } +export function isReadableStreamDisturbed(stream: ReadableStream): boolean { + assert(isReadableStream(stream)); + return stream[sym.disturbed] ? true : false; +} + export function isTransformStream( x: unknown ): x is TransformStreamImpl { diff --git a/cli/js/web/timers.ts b/cli/js/web/timers.ts index 90b6bba9494ef9..245d6a8c7153c1 100644 --- a/cli/js/web/timers.ts +++ b/cli/js/web/timers.ts @@ -234,23 +234,23 @@ function setTimer( } export function setTimeout( + this: unknown, cb: (...args: Args) => void, delay = 0, ...args: Args ): number { checkBigInt(delay); - // @ts-ignore checkThis(this); return setTimer(cb, delay, args, false); } export function setInterval( + this: unknown, cb: (...args: Args) => void, delay = 0, ...args: Args ): number { checkBigInt(delay); - // @ts-ignore checkThis(this); return setTimer(cb, delay, args, true); } diff --git a/cli/js/web/util.ts b/cli/js/web/util.ts index 0ab367554a556d..53ff8ef2273e95 100644 --- a/cli/js/web/util.ts +++ b/cli/js/web/util.ts @@ -189,3 +189,22 @@ export function defineEnumerableProps( Reflect.defineProperty(Ctor.prototype, prop, { enumerable: true }); } } + +// @internal +export function getHeaderValueParams(value: string): Map { + const params = new Map(); + // Forced to do so for some Map constructor param mismatch + value + .split(";") + .slice(1) + .map((s): string[] => s.trim().split("=")) + .filter((arr): boolean => arr.length > 1) + .map(([k, v]): [string, string] => [k, v.replace(/^"([^"]*)"$/, "$1")]) + .forEach(([k, v]): Map => params.set(k, v)); + return params; +} + +// @internal +export function hasHeaderValueOf(s: string, value: string): boolean { + return new RegExp(`^${value}[\t\s]*;?`).test(s); +} diff --git a/cli/main.rs b/cli/main.rs index 5811f4a9ea0068..3ea726b58b5279 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -1,6 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. #![deny(warnings)] +extern crate dissimilar; #[macro_use] extern crate lazy_static; #[macro_use] @@ -25,6 +26,7 @@ mod checksum; pub mod colors; pub mod deno_dir; pub mod diagnostics; +mod diff; mod disk_cache; mod doc; mod file_fetcher; @@ -72,17 +74,14 @@ use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; use crate::fs as deno_fs; use crate::global_state::GlobalState; -use crate::import_map::ImportMap; use crate::msg::MediaType; use crate::op_error::OpError; -use crate::ops::io::get_stdio; use crate::permissions::Permissions; -use crate::state::exit_unstable; -use crate::state::State; use crate::tsc::TargetLib; use crate::worker::MainWorker; use deno_core::v8_set_flags; use deno_core::ErrBox; +use deno_core::EsIsolate; use deno_core::ModuleSpecifier; use flags::DenoSubcommand; use flags::Flags; @@ -139,28 +138,17 @@ fn write_to_stdout_ignore_sigpipe(bytes: &[u8]) -> Result<(), std::io::Error> { } } -fn create_main_worker( - global_state: GlobalState, - main_module: ModuleSpecifier, -) -> Result { - let state = State::new(global_state, None, main_module, false)?; - - let mut worker = MainWorker::new( - "main".to_string(), - startup_data::deno_isolate_init(), - state, - ); - - { - let (stdin, stdout, stderr) = get_stdio(); - let mut t = worker.resource_table.borrow_mut(); - t.add("stdin", Box::new(stdin)); - t.add("stdout", Box::new(stdout)); - t.add("stderr", Box::new(stderr)); +fn write_lockfile(global_state: GlobalState) -> Result<(), std::io::Error> { + if global_state.flags.lock_write { + if let Some(ref lockfile) = global_state.lockfile { + let g = lockfile.lock().unwrap(); + g.write()?; + } else { + eprintln!("--lock flag must be specified when using --lock-write"); + std::process::exit(11); + } } - - worker.execute("bootstrap.mainRuntime()")?; - Ok(worker) + Ok(()) } fn print_cache_info(state: &GlobalState) { @@ -207,16 +195,21 @@ async fn print_file_info( ); let module_specifier_ = module_specifier.clone(); + global_state - .clone() - .fetch_compiled_module( - module_specifier_, + .prepare_module_load( + module_specifier_.clone(), None, TargetLib::Main, Permissions::allow_all(), false, + global_state.maybe_import_map.clone(), ) .await?; + global_state + .clone() + .fetch_compiled_module(module_specifier_, None) + .await?; if out.media_type == msg::MediaType::TypeScript || (out.media_type == msg::MediaType::JavaScript @@ -246,7 +239,10 @@ async fn print_file_info( ); } - if let Some(deps) = worker.isolate.modules.deps(&module_specifier) { + let es_state_rc = EsIsolate::state(&worker.isolate); + let es_state = es_state_rc.borrow(); + + if let Some(deps) = es_state.modules.deps(&module_specifier) { println!("{}{}", colors::bold("deps:\n".to_string()), deps.name); if let Some(ref depsdeps) = deps.deps { for d in depsdeps { @@ -294,7 +290,7 @@ async fn info_command( } let main_module = ModuleSpecifier::resolve_url_or_path(&file.unwrap())?; - let mut worker = create_main_worker(global_state, main_module.clone())?; + let mut worker = MainWorker::create(global_state, main_module.clone())?; worker.preload_module(&main_module).await?; print_file_info(&worker, main_module.clone()).await } @@ -310,27 +306,84 @@ async fn install_command( installer::install(flags, &module_url, args, name, root, force).await } +async fn lint_command(flags: Flags, files: Vec) -> Result<(), ErrBox> { + let global_state = GlobalState::new(flags)?; + + // TODO(bartlomieju): refactor, it's non-sense to create + // state just to perform unstable check... + use crate::state::State; + let state = State::new( + global_state.clone(), + None, + ModuleSpecifier::resolve_url("file:///dummy.ts").unwrap(), + None, + true, + )?; + + state.check_unstable("lint"); + + let mut error_counts = 0; + + for file in files { + let specifier = ModuleSpecifier::resolve_url_or_path(&file)?; + let source_file = global_state + .file_fetcher + .fetch_source_file(&specifier, None, Permissions::allow_all()) + .await?; + let source_code = String::from_utf8(source_file.source_code)?; + + let mut linter = deno_lint::linter::Linter::default(); + let lint_rules = deno_lint::rules::get_all_rules(); + + let file_diagnostics = linter.lint(file, source_code, lint_rules)?; + + error_counts += file_diagnostics.len(); + for d in file_diagnostics.iter() { + let pretty_message = format!( + "({}) {}", + colors::gray(d.code.to_string()), + d.message.clone() + ); + eprintln!( + "{}\n", + fmt_errors::format_stack( + true, + pretty_message, + Some(d.line_src.clone()), + Some(d.location.col as i64), + Some((d.location.col + d.snippet_length) as i64), + &[fmt_errors::format_location( + d.location.filename.clone(), + d.location.line as i64, + d.location.col as i64, + )], + 0 + ) + ); + } + } + + if error_counts > 0 { + eprintln!("Found {} problems", error_counts); + std::process::exit(1); + } + + Ok(()) +} + async fn cache_command(flags: Flags, files: Vec) -> Result<(), ErrBox> { let main_module = ModuleSpecifier::resolve_url_or_path("./__$deno$fetch.ts").unwrap(); let global_state = GlobalState::new(flags)?; let mut worker = - create_main_worker(global_state.clone(), main_module.clone())?; + MainWorker::create(global_state.clone(), main_module.clone())?; for file in files { let specifier = ModuleSpecifier::resolve_url_or_path(&file)?; worker.preload_module(&specifier).await.map(|_| ())?; } - if global_state.flags.lock_write { - if let Some(ref lockfile) = global_state.lockfile { - let g = lockfile.lock().unwrap(); - g.write()?; - } else { - eprintln!("--lock flag must be specified when using --lock-write"); - std::process::exit(11); - } - } + write_lockfile(global_state)?; Ok(()) } @@ -339,14 +392,22 @@ async fn eval_command( flags: Flags, code: String, as_typescript: bool, + print: bool, ) -> Result<(), ErrBox> { // Force TypeScript compile. let main_module = ModuleSpecifier::resolve_url_or_path("./__$deno$eval.ts").unwrap(); let global_state = GlobalState::new(flags)?; - let mut worker = create_main_worker(global_state, main_module.clone())?; + let mut worker = MainWorker::create(global_state, main_module.clone())?; let main_module_url = main_module.as_url().to_owned(); // Create a dummy source file. + let source_code = if print { + "console.log(".to_string() + &code + ")" + } else { + code.clone() + } + .into_bytes(); + let source_file = SourceFile { filename: main_module_url.to_file_path().unwrap(), url: main_module_url, @@ -357,7 +418,7 @@ async fn eval_command( } else { MediaType::JavaScript }, - source_code: code.clone().into_bytes(), + source_code, }; // Save our fake file into file fetcher cache // to allow module access by TS compiler (e.g. op_fetch_source_files) @@ -380,43 +441,78 @@ async fn bundle_command( source_file: String, out_file: Option, ) -> Result<(), ErrBox> { - let mut module_name = ModuleSpecifier::resolve_url_or_path(&source_file)?; - let url = module_name.as_url(); + let mut module_specifier = + ModuleSpecifier::resolve_url_or_path(&source_file)?; + let url = module_specifier.as_url(); // TODO(bartlomieju): fix this hack in ModuleSpecifier if url.scheme() == "file" { let a = deno_fs::normalize_path(&url.to_file_path().unwrap()); let u = Url::from_file_path(a).unwrap(); - module_name = ModuleSpecifier::from(u) + module_specifier = ModuleSpecifier::from(u) } debug!(">>>>> bundle START"); let compiler_config = tsc::CompilerConfig::load(flags.config_path.clone())?; - let maybe_import_map = match flags.import_map_path.as_ref() { - None => None, - Some(file_path) => { - if !flags.unstable { - exit_unstable("--importmap") - } - Some(ImportMap::load(file_path)?) - } - }; - let global_state = GlobalState::new(flags)?; - let bundle_result = tsc::bundle( + info!("Bundling {}", module_specifier.to_string()); + + let output = tsc::bundle( &global_state, compiler_config, - module_name, - maybe_import_map, - out_file, + module_specifier, + global_state.maybe_import_map.clone(), global_state.flags.unstable, ) - .await; + .await?; debug!(">>>>> bundle END"); - bundle_result + + if let Some(out_file_) = out_file.as_ref() { + info!("Emitting bundle to {:?}", out_file_); + let output_bytes = output.as_bytes(); + let output_len = output_bytes.len(); + deno_fs::write_file(out_file_, output_bytes, 0o666)?; + info!("{} emitted.", human_size(output_len as f64)); + } else { + println!("{}", output); + } + Ok(()) +} + +fn human_size(bytse: f64) -> String { + let negative = if bytse.is_sign_positive() { "" } else { "-" }; + let bytse = bytse.abs(); + let units = ["Bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; + if bytse < 1_f64 { + return format!("{}{} {}", negative, bytse, "Bytes"); + } + let delimiter = 1024_f64; + let exponent = std::cmp::min( + (bytse.ln() / delimiter.ln()).floor() as i32, + (units.len() - 1) as i32, + ); + let pretty_bytes = format!("{:.2}", bytse / delimiter.powi(exponent)) + .parse::() + .unwrap() + * 1_f64; + let unit = units[exponent as usize]; + format!("{}{} {}", negative, pretty_bytes, unit) +} + +#[test] +fn human_size_test() { + assert_eq!(human_size(16_f64), "16 Bytes"); + assert_eq!(human_size((16 * 1024) as f64), "16 KB"); + assert_eq!(human_size((16 * 1024 * 1024) as f64), "16 MB"); + assert_eq!(human_size(16_f64 * 1024_f64.powf(3.0)), "16 GB"); + assert_eq!(human_size(16_f64 * 1024_f64.powf(4.0)), "16 TB"); + assert_eq!(human_size(16_f64 * 1024_f64.powf(5.0)), "16 PB"); + assert_eq!(human_size(16_f64 * 1024_f64.powf(6.0)), "16 EB"); + assert_eq!(human_size(16_f64 * 1024_f64.powf(7.0)), "16 ZB"); + assert_eq!(human_size(16_f64 * 1024_f64.powf(8.0)), "16 YB"); } async fn doc_command( @@ -493,7 +589,7 @@ async fn run_repl(flags: Flags) -> Result<(), ErrBox> { let main_module = ModuleSpecifier::resolve_url_or_path("./__$deno$repl.ts").unwrap(); let global_state = GlobalState::new(flags)?; - let mut worker = create_main_worker(global_state, main_module)?; + let mut worker = MainWorker::create(global_state, main_module)?; loop { (&mut *worker).await?; } @@ -503,21 +599,13 @@ async fn run_command(flags: Flags, script: String) -> Result<(), ErrBox> { let global_state = GlobalState::new(flags.clone())?; let main_module = ModuleSpecifier::resolve_url_or_path(&script).unwrap(); let mut worker = - create_main_worker(global_state.clone(), main_module.clone())?; + MainWorker::create(global_state.clone(), main_module.clone())?; debug!("main_module {}", main_module); worker.execute_module(&main_module).await?; + write_lockfile(global_state)?; worker.execute("window.dispatchEvent(new Event('load'))")?; (&mut *worker).await?; worker.execute("window.dispatchEvent(new Event('unload'))")?; - if global_state.flags.lock_write { - if let Some(ref lockfile) = global_state.lockfile { - let g = lockfile.lock().unwrap(); - g.write()?; - } else { - eprintln!("--lock flag must be specified when using --lock-write"); - std::process::exit(11); - } - } Ok(()) } @@ -550,7 +638,7 @@ async fn test_command( let main_module = ModuleSpecifier::resolve_url(&test_file_url.to_string()).unwrap(); let mut worker = - create_main_worker(global_state.clone(), main_module.clone())?; + MainWorker::create(global_state.clone(), main_module.clone())?; // Create a dummy source file. let source_file = SourceFile { filename: test_file_url.to_file_path().unwrap(), @@ -606,9 +694,10 @@ pub fn main() { filter, } => doc_command(flags, source_file, json, filter).boxed_local(), DenoSubcommand::Eval { + print, code, as_typescript, - } => eval_command(flags, code, as_typescript).boxed_local(), + } => eval_command(flags, code, as_typescript, print).boxed_local(), DenoSubcommand::Cache { files } => { cache_command(flags, files).boxed_local() } @@ -625,6 +714,7 @@ pub fn main() { } => { install_command(flags, module_url, args, name, root, force).boxed_local() } + DenoSubcommand::Lint { files } => lint_command(flags, files).boxed_local(), DenoSubcommand::Repl => run_repl(flags).boxed_local(), DenoSubcommand::Run { script } => run_command(flags, script).boxed_local(), DenoSubcommand::Test { diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 3a59a537d7f154..519f443ff991d5 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -1,5 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::doc::Location; use crate::file_fetcher::map_file_extension; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; @@ -9,7 +10,7 @@ use crate::op_error::OpError; use crate::permissions::Permissions; use crate::swc_util::analyze_dependencies_and_references; use crate::swc_util::TsReferenceKind; -use crate::tsc::get_available_libs; +use crate::tsc::AVAILABLE_LIBS; use deno_core::ErrBox; use deno_core::ModuleSpecifier; use futures::stream::FuturesUnordered; @@ -24,6 +25,18 @@ use std::hash::BuildHasher; use std::path::PathBuf; use std::pin::Pin; +// TODO(bartlomieju): it'd be great if this function returned +// more structured data and possibly format the same as TS diagnostics. +/// Decorate error with location of import that caused the error. +fn err_with_location(e: ErrBox, location: &Location) -> ErrBox { + let location_str = format!( + "\nImported from \"{}:{}\"", + location.filename, location.line + ); + let err_str = e.to_string(); + OpError::other(format!("{}{}", err_str, location_str)).into() +} + fn serialize_module_specifier( spec: &ModuleSpecifier, s: S, @@ -138,8 +151,9 @@ impl ModuleGraphLoader { pub async fn add_to_graph( &mut self, specifier: &ModuleSpecifier, + maybe_referrer: Option, ) -> Result<(), ErrBox> { - self.download_module(specifier.clone(), None)?; + self.download_module(specifier.clone(), maybe_referrer)?; loop { let (specifier, source_file) = @@ -196,6 +210,7 @@ impl ModuleGraphLoader { let (import_descs, ref_descs) = analyze_dependencies_and_references( &specifier, + map_file_extension(&PathBuf::from(&specifier)), &source_code, self.analyze_dynamic_imports, )?; @@ -238,10 +253,8 @@ impl ModuleGraphLoader { imports.push(import_descriptor); } - let available_libs = get_available_libs(); - for ref_desc in ref_descs { - if available_libs.contains(&ref_desc.specifier) { + if AVAILABLE_LIBS.contains(&ref_desc.specifier.as_str()) { continue; } @@ -409,6 +422,7 @@ impl ModuleGraphLoader { let (import_descs, ref_descs) = analyze_dependencies_and_references( &module_specifier.to_string(), + source_file.media_type, &source_code, self.analyze_dynamic_imports, )?; @@ -444,31 +458,33 @@ impl ModuleGraphLoader { let import_descriptor = ImportDescriptor { specifier: import_desc.specifier.to_string(), resolved_specifier, - type_directive: import_desc.deno_types, + type_directive: import_desc.deno_types.clone(), resolved_type_directive, }; - self.download_module( - import_descriptor.resolved_specifier.clone(), - Some(module_specifier.clone()), - )?; + self + .download_module( + import_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + ) + .map_err(|e| err_with_location(e, &import_desc.location))?; if let Some(type_dir_url) = import_descriptor.resolved_type_directive.as_ref() { - self.download_module( - type_dir_url.clone(), - Some(module_specifier.clone()), - )?; + self + .download_module( + type_dir_url.clone(), + Some(module_specifier.clone()), + ) + .map_err(|e| err_with_location(e, &import_desc.location))?; } imports.push(import_descriptor); } - let available_libs = get_available_libs(); - for ref_desc in ref_descs { - if available_libs.contains(&ref_desc.specifier) { + if AVAILABLE_LIBS.contains(&ref_desc.specifier.as_str()) { continue; } @@ -482,10 +498,12 @@ impl ModuleGraphLoader { resolved_specifier, }; - self.download_module( - reference_descriptor.resolved_specifier.clone(), - Some(module_specifier.clone()), - )?; + self + .download_module( + reference_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + ) + .map_err(|e| err_with_location(e, &ref_desc.location))?; match ref_desc.kind { TsReferenceKind::Lib => { @@ -537,7 +555,7 @@ mod tests { false, false, ); - graph_loader.add_to_graph(&module_specifier).await?; + graph_loader.add_to_graph(&module_specifier, None).await?; Ok(graph_loader.get_graph()) } @@ -786,4 +804,23 @@ mod tests { ); drop(http_server_guard); } + + #[tokio::test] + async fn source_graph_different_langs() { + let http_server_guard = crate::test_util::http_server(); + + // ModuleGraphLoader was mistakenly parsing this file as TSX + // https://github.com/denoland/deno/issues/5867 + + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/ts_with_generic.ts", + ) + .unwrap(); + + build_graph(&module_specifier) + .await + .expect("Failed to build graph"); + + drop(http_server_guard); + } } diff --git a/cli/ops/dispatch_json.rs b/cli/ops/dispatch_json.rs index 6125ea39bb14bb..97a02991f5b169 100644 --- a/cli/ops/dispatch_json.rs +++ b/cli/ops/dispatch_json.rs @@ -1,7 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. use crate::op_error::OpError; use deno_core::Buf; -use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::Op; use deno_core::ZeroCopyBuf; use futures::future::FutureExt; @@ -46,14 +46,17 @@ struct AsyncArgs { pub fn json_op( d: D, -) -> impl Fn(&mut CoreIsolate, &[u8], Option) -> Op +) -> impl Fn(&mut CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op where - D: - Fn(&mut CoreIsolate, Value, Option) -> Result, + D: Fn( + &mut CoreIsolateState, + Value, + &mut [ZeroCopyBuf], + ) -> Result, { - move |isolate: &mut CoreIsolate, + move |isolate_state: &mut CoreIsolateState, control: &[u8], - zero_copy: Option| { + zero_copy: &mut [ZeroCopyBuf]| { let async_args: AsyncArgs = match serde_json::from_slice(control) { Ok(args) => args, Err(e) => { @@ -66,7 +69,7 @@ where let result = serde_json::from_slice(control) .map_err(OpError::from) - .and_then(|args| d(isolate, args, zero_copy)); + .and_then(|args| d(isolate_state, args, zero_copy)); // Convert to Op match result { diff --git a/cli/ops/dispatch_minimal.rs b/cli/ops/dispatch_minimal.rs index ac98ea5896486e..eac5ad05573358 100644 --- a/cli/ops/dispatch_minimal.rs +++ b/cli/ops/dispatch_minimal.rs @@ -7,7 +7,7 @@ use crate::op_error::OpError; use byteorder::{LittleEndian, WriteBytesExt}; use deno_core::Buf; -use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::Op; use deno_core::ZeroCopyBuf; use futures::future::FutureExt; @@ -116,13 +116,13 @@ fn test_parse_min_record() { pub fn minimal_op( d: D, -) -> impl Fn(&mut CoreIsolate, &[u8], Option) -> Op +) -> impl Fn(&mut CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op where - D: Fn(&mut CoreIsolate, bool, i32, Option) -> MinimalOp, + D: Fn(&mut CoreIsolateState, bool, i32, &mut [ZeroCopyBuf]) -> MinimalOp, { - move |isolate: &mut CoreIsolate, + move |isolate_state: &mut CoreIsolateState, control: &[u8], - zero_copy: Option| { + zero_copy: &mut [ZeroCopyBuf]| { let mut record = match parse_min_record(control) { Some(r) => r, None => { @@ -138,7 +138,7 @@ where }; let is_sync = record.promise_id == 0; let rid = record.arg; - let min_op = d(isolate, is_sync, rid, zero_copy); + let min_op = d(isolate_state, is_sync, rid, zero_copy); match min_op { MinimalOp::Sync(sync_result) => Op::Sync(match sync_result { diff --git a/cli/ops/errors.rs b/cli/ops/errors.rs index 766c130e2c2cd9..ade125b1a1552d 100644 --- a/cli/ops/errors.rs +++ b/cli/ops/errors.rs @@ -31,7 +31,7 @@ struct ApplySourceMap { fn op_apply_source_map( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ApplySourceMap = serde_json::from_value(args)?; @@ -55,7 +55,7 @@ fn op_apply_source_map( fn op_format_diagnostic( _state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let diagnostic = serde_json::from_value::(args)?; Ok(JsonOp::Sync(json!(diagnostic.to_string()))) diff --git a/cli/ops/fetch.rs b/cli/ops/fetch.rs index 596c9a2fd2058d..5a646325edb6c3 100644 --- a/cli/ops/fetch.rs +++ b/cli/ops/fetch.rs @@ -5,6 +5,7 @@ use crate::http_util::{create_http_client, HttpBody}; use crate::op_error::OpError; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ZeroCopyBuf; use futures::future::FutureExt; use http::header::HeaderName; @@ -24,10 +25,10 @@ struct FetchArgs { } pub fn op_fetch( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - data: Option, + data: &mut [ZeroCopyBuf], ) -> Result { let args: FetchArgs = serde_json::from_value(args)?; let url = args.url; @@ -56,8 +57,10 @@ pub fn op_fetch( let mut request = client.request(method, url_); - if let Some(buf) = data { - request = request.body(Vec::from(&*buf)); + match data.len() { + 0 => {} + 1 => request = request.body(Vec::from(&*data[0])), + _ => panic!("Invalid number of arguments"), } for (key, value) in args.headers { @@ -67,7 +70,7 @@ pub fn op_fetch( } debug!("Before fetch {}", url); - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); let future = async move { let res = request.send().await?; debug!("Fetch response {}", url); diff --git a/cli/ops/fs.rs b/cli/ops/fs.rs index 9d5be30778ac2f..63b3ad7f56ea7b 100644 --- a/cli/ops/fs.rs +++ b/cli/ops/fs.rs @@ -3,11 +3,11 @@ use super::dispatch_json::{blocking_json, Deserialize, JsonOp, Value}; use super::io::std_file_resource; use super::io::{FileMetadata, StreamResource, StreamResourceHolder}; -use crate::fs::resolve_from_cwd; use crate::op_error::OpError; use crate::ops::dispatch_json::JsonResult; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ZeroCopyBuf; use futures::future::FutureExt; use std::convert::From; @@ -69,14 +69,14 @@ struct OpenOptions { } fn op_open( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: OpenArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; - let resource_table = isolate.resource_table.clone(); + let path = Path::new(&args.path).to_path_buf(); + let resource_table = isolate_state.resource_table.clone(); let mut open_options = std::fs::OpenOptions::new(); @@ -152,10 +152,10 @@ struct SeekArgs { } fn op_seek( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { use std::io::{Seek, SeekFrom}; let args: SeekArgs = serde_json::from_value(args)?; @@ -175,7 +175,7 @@ fn op_seek( } }; - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); let is_sync = args.promise_id.is_none(); if is_sync { @@ -212,7 +212,7 @@ struct UmaskArgs { fn op_umask( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.umask"); let args: UmaskArgs = serde_json::from_value(args)?; @@ -250,7 +250,7 @@ struct ChdirArgs { fn op_chdir( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ChdirArgs = serde_json::from_value(args)?; let d = PathBuf::from(&args.directory); @@ -271,10 +271,10 @@ struct MkdirArgs { fn op_mkdir( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: MkdirArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = Path::new(&args.path).to_path_buf(); let mode = args.mode.unwrap_or(0o777) & 0o777; state.check_write(&path)?; @@ -305,10 +305,10 @@ struct ChmodArgs { fn op_chmod( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ChmodArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = Path::new(&args.path).to_path_buf(); let mode = args.mode & 0o777; state.check_write(&path)?; @@ -345,10 +345,10 @@ struct ChownArgs { fn op_chown( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ChownArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = Path::new(&args.path).to_path_buf(); state.check_write(&path)?; @@ -384,10 +384,10 @@ struct RemoveArgs { fn op_remove( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: RemoveArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = PathBuf::from(&args.path); let recursive = args.recursive; state.check_write(&path)?; @@ -417,8 +417,11 @@ fn op_remove( std::fs::remove_file(&path)?; } } - } else { + } else if file_type.is_dir() { std::fs::remove_dir(&path)?; + } else { + // pipes, sockets, etc... + std::fs::remove_file(&path)?; } Ok(json!({})) }) @@ -435,11 +438,11 @@ struct CopyFileArgs { fn op_copy_file( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: CopyFileArgs = serde_json::from_value(args)?; - let from = resolve_from_cwd(Path::new(&args.from))?; - let to = resolve_from_cwd(Path::new(&args.to))?; + let from = PathBuf::from(&args.from); + let to = PathBuf::from(&args.to); state.check_read(&from)?; state.check_write(&to)?; @@ -529,10 +532,10 @@ struct StatArgs { fn op_stat( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: StatArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = PathBuf::from(&args.path); let lstat = args.lstat; state.check_read(&path)?; @@ -559,12 +562,15 @@ struct RealpathArgs { fn op_realpath( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: RealpathArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = PathBuf::from(&args.path); state.check_read(&path)?; + if path.is_relative() { + state.check_read_blind(¤t_dir()?, "CWD")?; + } let is_sync = args.promise_id.is_none(); blocking_json(is_sync, move || { @@ -591,10 +597,10 @@ struct ReadDirArgs { fn op_read_dir( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ReadDirArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = PathBuf::from(&args.path); state.check_read(&path)?; @@ -634,11 +640,11 @@ struct RenameArgs { fn op_rename( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: RenameArgs = serde_json::from_value(args)?; - let oldpath = resolve_from_cwd(Path::new(&args.oldpath))?; - let newpath = resolve_from_cwd(Path::new(&args.newpath))?; + let oldpath = PathBuf::from(&args.oldpath); + let newpath = PathBuf::from(&args.newpath); state.check_read(&oldpath)?; state.check_write(&oldpath)?; @@ -663,12 +669,12 @@ struct LinkArgs { fn op_link( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.link"); let args: LinkArgs = serde_json::from_value(args)?; - let oldpath = resolve_from_cwd(Path::new(&args.oldpath))?; - let newpath = resolve_from_cwd(Path::new(&args.newpath))?; + let oldpath = PathBuf::from(&args.oldpath); + let newpath = PathBuf::from(&args.newpath); state.check_read(&oldpath)?; state.check_write(&newpath)?; @@ -701,12 +707,12 @@ struct SymlinkOptions { fn op_symlink( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.symlink"); let args: SymlinkArgs = serde_json::from_value(args)?; - let oldpath = resolve_from_cwd(Path::new(&args.oldpath))?; - let newpath = resolve_from_cwd(Path::new(&args.newpath))?; + let oldpath = PathBuf::from(&args.oldpath); + let newpath = PathBuf::from(&args.newpath); state.check_write(&newpath)?; @@ -761,10 +767,10 @@ struct ReadLinkArgs { fn op_read_link( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ReadLinkArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = PathBuf::from(&args.path); state.check_read(&path)?; @@ -788,10 +794,10 @@ struct TruncateArgs { fn op_truncate( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: TruncateArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = PathBuf::from(&args.path); let len = args.len; state.check_write(&path)?; @@ -862,11 +868,11 @@ struct MakeTempArgs { fn op_make_temp_dir( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: MakeTempArgs = serde_json::from_value(args)?; - let dir = args.dir.map(|s| resolve_from_cwd(Path::new(&s)).unwrap()); + let dir = args.dir.map(|s| PathBuf::from(&s)); let prefix = args.prefix.map(String::from); let suffix = args.suffix.map(String::from); @@ -893,11 +899,11 @@ fn op_make_temp_dir( fn op_make_temp_file( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: MakeTempArgs = serde_json::from_value(args)?; - let dir = args.dir.map(|s| resolve_from_cwd(Path::new(&s)).unwrap()); + let dir = args.dir.map(|s| PathBuf::from(&s)); let prefix = args.prefix.map(String::from); let suffix = args.suffix.map(String::from); @@ -926,19 +932,19 @@ fn op_make_temp_file( struct UtimeArgs { promise_id: Option, path: String, - atime: u64, - mtime: u64, + atime: i64, + mtime: i64, } fn op_utime( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.utime"); let args: UtimeArgs = serde_json::from_value(args)?; - let path = resolve_from_cwd(Path::new(&args.path))?; + let path = PathBuf::from(&args.path); state.check_write(&path)?; @@ -953,10 +959,10 @@ fn op_utime( fn op_cwd( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let path = current_dir()?; - state.check_read(&path)?; + state.check_read_blind(&path, "CWD")?; let path_str = into_string(path.into_os_string())?; Ok(JsonOp::Sync(json!(path_str))) } diff --git a/cli/ops/fs_events.rs b/cli/ops/fs_events.rs index 56ed556f4f7a2a..b9d54fc3e5bd6e 100644 --- a/cli/ops/fs_events.rs +++ b/cli/ops/fs_events.rs @@ -3,6 +3,7 @@ use super::dispatch_json::{Deserialize, JsonOp, Value}; use crate::op_error::OpError; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ErrBox; use deno_core::ZeroCopyBuf; use futures::future::poll_fn; @@ -62,10 +63,10 @@ impl From for FsEvent { } pub fn op_fs_events_open( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { #[derive(Deserialize)] struct OpenArgs { @@ -94,23 +95,23 @@ pub fn op_fs_events_open( watcher.watch(path, recursive_mode).map_err(ErrBox::from)?; } let resource = FsEventsResource { watcher, receiver }; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let rid = resource_table.add("fsEvents", Box::new(resource)); Ok(JsonOp::Sync(json!(rid))) } pub fn op_fs_events_poll( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { #[derive(Deserialize)] struct PollArgs { rid: u32, } let PollArgs { rid } = serde_json::from_value(args)?; - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); let f = poll_fn(move |cx| { let mut resource_table = resource_table.borrow_mut(); let watcher = resource_table diff --git a/cli/ops/io.rs b/cli/ops/io.rs index 705083e15cacbf..0e007ad1a5f775 100644 --- a/cli/ops/io.rs +++ b/cli/ops/io.rs @@ -3,6 +3,7 @@ use crate::http_util::HttpBody; use crate::op_error::OpError; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ResourceTable; use deno_core::ZeroCopyBuf; use futures::future::poll_fn; @@ -206,19 +207,19 @@ impl DenoAsyncRead for StreamResource { } pub fn op_read( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, is_sync: bool, rid: i32, - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> MinimalOp { debug!("read rid={}", rid); - if zero_copy.is_none() { - return MinimalOp::Sync(Err(no_buffer_specified())); + match zero_copy.len() { + 0 => return MinimalOp::Sync(Err(no_buffer_specified())), + 1 => {} + _ => panic!("Invalid number of arguments"), } - let resource_table = isolate.resource_table.clone(); - - let mut buf = zero_copy.unwrap(); + let resource_table = isolate_state.resource_table.clone(); if is_sync { MinimalOp::Sync({ @@ -228,7 +229,7 @@ pub fn op_read( Ok(std_file) => { use std::io::Read; std_file - .read(&mut buf) + .read(&mut zero_copy[0]) .map(|n: usize| n as i32) .map_err(OpError::from) } @@ -238,6 +239,7 @@ pub fn op_read( }) }) } else { + let mut zero_copy = zero_copy[0].clone(); MinimalOp::Async( poll_fn(move |cx| { let mut resource_table = resource_table.borrow_mut(); @@ -248,7 +250,7 @@ pub fn op_read( let mut task_tracker_id: Option = None; let nread = match resource_holder .resource - .poll_read(cx, &mut buf.as_mut()[..]) + .poll_read(cx, &mut zero_copy) .map_err(OpError::from) { Poll::Ready(t) => { @@ -330,28 +332,28 @@ impl DenoAsyncWrite for StreamResource { } pub fn op_write( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, is_sync: bool, rid: i32, - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> MinimalOp { debug!("write rid={}", rid); - if zero_copy.is_none() { - return MinimalOp::Sync(Err(no_buffer_specified())); + match zero_copy.len() { + 0 => return MinimalOp::Sync(Err(no_buffer_specified())), + 1 => {} + _ => panic!("Invalid number of arguments"), } - let buf = zero_copy.unwrap(); - if is_sync { MinimalOp::Sync({ // First we look up the rid in the resource table. - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); std_file_resource(&mut resource_table, rid as u32, move |r| match r { Ok(std_file) => { use std::io::Write; std_file - .write(&buf) + .write(&zero_copy[0]) .map(|nwritten: usize| nwritten as i32) .map_err(OpError::from) } @@ -361,7 +363,8 @@ pub fn op_write( }) }) } else { - let resource_table = isolate.resource_table.clone(); + let zero_copy = zero_copy[0].clone(); + let resource_table = isolate_state.resource_table.clone(); MinimalOp::Async( async move { let nwritten = poll_fn(|cx| { @@ -369,7 +372,7 @@ pub fn op_write( let resource_holder = resource_table .get_mut::(rid as u32) .ok_or_else(OpError::bad_resource_id)?; - resource_holder.resource.poll_write(cx, &buf.as_ref()[..]) + resource_holder.resource.poll_write(cx, &zero_copy) }) .await?; diff --git a/cli/ops/net.rs b/cli/ops/net.rs index 59707e291ff287..5ccb62d6c807d0 100644 --- a/cli/ops/net.rs +++ b/cli/ops/net.rs @@ -5,6 +5,7 @@ use crate::op_error::OpError; use crate::resolve_addr::resolve_addr; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ResourceTable; use deno_core::ZeroCopyBuf; use futures::future::poll_fn; @@ -37,12 +38,12 @@ struct AcceptArgs { } fn accept_tcp( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, args: AcceptArgs, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let rid = args.rid as u32; - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); let op = async move { let accept_fut = poll_fn(|cx| { @@ -97,16 +98,16 @@ fn accept_tcp( } fn op_accept( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, args: Value, - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: AcceptArgs = serde_json::from_value(args)?; match args.transport.as_str() { - "tcp" => accept_tcp(isolate, args, zero_copy), + "tcp" => accept_tcp(isolate_state, args, zero_copy), #[cfg(unix)] - "unix" => net_unix::accept_unix(isolate, args.rid as u32, zero_copy), + "unix" => net_unix::accept_unix(isolate_state, args.rid as u32, zero_copy), _ => Err(OpError::other(format!( "Unsupported transport protocol {}", args.transport @@ -121,16 +122,17 @@ struct ReceiveArgs { } fn receive_udp( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, args: ReceiveArgs, - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> Result { - let mut buf = zero_copy.unwrap(); + assert_eq!(zero_copy.len(), 1, "Invalid number of arguments"); + let mut zero_copy = zero_copy[0].clone(); let rid = args.rid as u32; - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); let op = async move { let receive_fut = poll_fn(|cx| { @@ -141,7 +143,9 @@ fn receive_udp( OpError::bad_resource("Socket has been closed".to_string()) })?; let socket = &mut resource.socket; - socket.poll_recv_from(cx, &mut buf).map_err(OpError::from) + socket + .poll_recv_from(cx, &mut zero_copy) + .map_err(OpError::from) }); let (size, remote_addr) = receive_fut.await?; Ok(json!({ @@ -158,18 +162,19 @@ fn receive_udp( } fn op_receive( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> Result { - assert!(zero_copy.is_some()); + assert_eq!(zero_copy.len(), 1, "Invalid number of arguments"); + let args: ReceiveArgs = serde_json::from_value(args)?; match args.transport.as_str() { - "udp" => receive_udp(isolate, state, args, zero_copy), + "udp" => receive_udp(isolate_state, state, args, zero_copy), #[cfg(unix)] "unixpacket" => { - net_unix::receive_unix_packet(isolate, args.rid as u32, zero_copy) + net_unix::receive_unix_packet(isolate_state, args.rid as u32, zero_copy) } _ => Err(OpError::other(format!( "Unsupported transport protocol {}", @@ -187,14 +192,15 @@ struct SendArgs { } fn op_send( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> Result { - assert!(zero_copy.is_some()); - let buf = zero_copy.unwrap(); - let resource_table = isolate.resource_table.clone(); + assert_eq!(zero_copy.len(), 1, "Invalid number of arguments"); + let zero_copy = zero_copy[0].clone(); + + let resource_table = isolate_state.resource_table.clone(); match serde_json::from_value(args)? { SendArgs { rid, @@ -212,7 +218,7 @@ fn op_send( })?; let socket = &mut resource.socket; let addr = resolve_addr(&args.hostname, args.port)?; - socket.send_to(&buf, addr).await?; + socket.send_to(&zero_copy, addr).await?; Ok(json!({})) }; @@ -236,7 +242,7 @@ fn op_send( let socket = &mut resource.socket; socket - .send_to(&buf, &resource.local_addr.as_pathname().unwrap()) + .send_to(&zero_copy, &resource.local_addr.as_pathname().unwrap()) .await?; Ok(json!({})) @@ -256,12 +262,12 @@ struct ConnectArgs { } fn op_connect( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); match serde_json::from_value(args)? { ConnectArgs { transport, @@ -302,6 +308,7 @@ fn op_connect( transport_args: ArgsEnum::Unix(args), } if transport == "unix" => { let address_path = net_unix::Path::new(&args.path); + state.check_unstable("Deno.connect"); state.check_read(&address_path)?; let op = async move { let path = args.path; @@ -341,10 +348,10 @@ struct ShutdownArgs { } fn op_shutdown( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.shutdown"); @@ -359,7 +366,7 @@ fn op_shutdown( _ => unimplemented!(), }; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let resource_holder = resource_table .get_mut::(rid) .ok_or_else(OpError::bad_resource_id)?; @@ -483,12 +490,12 @@ fn listen_udp( } fn op_listen( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); match serde_json::from_value(args)? { ListenArgs { transport, @@ -524,6 +531,9 @@ fn op_listen( transport, transport_args: ArgsEnum::Unix(args), } if transport == "unix" || transport == "unixpacket" => { + if transport == "unix" { + state.check_unstable("Deno.listen"); + } if transport == "unixpacket" { state.check_unstable("Deno.listenDatagram"); } diff --git a/cli/ops/net_unix.rs b/cli/ops/net_unix.rs index 4b09c2fdb2fc4b..0b2ceb75a1d6bc 100644 --- a/cli/ops/net_unix.rs +++ b/cli/ops/net_unix.rs @@ -1,7 +1,7 @@ use super::dispatch_json::{Deserialize, JsonOp}; use super::io::{StreamResource, StreamResourceHolder}; use crate::op_error::OpError; -use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ResourceTable; use deno_core::ZeroCopyBuf; use futures::future::FutureExt; @@ -27,11 +27,11 @@ pub struct UnixListenArgs { } pub fn accept_unix( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, rid: u32, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); { let _ = resource_table .borrow() @@ -78,12 +78,13 @@ pub fn accept_unix( } pub fn receive_unix_packet( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, rid: u32, - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> Result { - let mut buf = zero_copy.unwrap(); - let resource_table = isolate.resource_table.clone(); + assert_eq!(zero_copy.len(), 1, "Invalid number of arguments"); + let mut zero_copy = zero_copy[0].clone(); + let resource_table = isolate_state.resource_table.clone(); let op = async move { let mut resource_table_ = resource_table.borrow_mut(); @@ -92,7 +93,7 @@ pub fn receive_unix_packet( .ok_or_else(|| { OpError::bad_resource("Socket has been closed".to_string()) })?; - let (size, remote_addr) = resource.socket.recv_from(&mut buf).await?; + let (size, remote_addr) = resource.socket.recv_from(&mut zero_copy).await?; Ok(json!({ "size": size, "remoteAddr": { diff --git a/cli/ops/os.rs b/cli/ops/os.rs index 6c18015207c09f..36cd99577d1d0e 100644 --- a/cli/ops/os.rs +++ b/cli/ops/os.rs @@ -29,7 +29,7 @@ struct GetDirArgs { fn op_get_dir( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.dir"); state.check_env()?; @@ -80,10 +80,10 @@ fn op_get_dir( fn op_exec_path( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let current_exe = env::current_exe().unwrap(); - state.check_read(¤t_exe)?; + state.check_read_blind(¤t_exe, "exec_path")?; // Now apply URL parser to current exe to get fully resolved path, otherwise // we might get `./` and `../` bits in `exec_path` let exe_url = Url::from_file_path(current_exe).unwrap(); @@ -100,7 +100,7 @@ struct SetEnv { fn op_set_env( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: SetEnv = serde_json::from_value(args)?; state.check_env()?; @@ -111,7 +111,7 @@ fn op_set_env( fn op_env( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_env()?; let v = env::vars().collect::>(); @@ -126,7 +126,7 @@ struct GetEnv { fn op_get_env( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: GetEnv = serde_json::from_value(args)?; state.check_env()?; @@ -145,7 +145,7 @@ struct Exit { fn op_exit( _s: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: Exit = serde_json::from_value(args)?; std::process::exit(args.code) @@ -154,7 +154,7 @@ fn op_exit( fn op_loadavg( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.loadavg"); state.check_env()?; @@ -171,7 +171,7 @@ fn op_loadavg( fn op_hostname( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.hostname"); state.check_env()?; @@ -182,7 +182,7 @@ fn op_hostname( fn op_os_release( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.osRelease"); state.check_env()?; diff --git a/cli/ops/permissions.rs b/cli/ops/permissions.rs index 3ccff4065690e8..a4ee4120e7c056 100644 --- a/cli/ops/permissions.rs +++ b/cli/ops/permissions.rs @@ -1,6 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. use super::dispatch_json::{Deserialize, JsonOp, Value}; -use crate::fs as deno_fs; use crate::op_error::OpError; use crate::state::State; use deno_core::CoreIsolate; @@ -29,26 +28,18 @@ struct PermissionArgs { path: Option, } -fn resolve_path(path: &str) -> String { - deno_fs::resolve_from_cwd(Path::new(path)) - .unwrap() - .to_str() - .unwrap() - .to_string() -} - pub fn op_query_permission( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: PermissionArgs = serde_json::from_value(args)?; let state = state.borrow(); - let resolved_path = args.path.as_deref().map(resolve_path); + let path = args.path.as_deref(); let perm = state.permissions.get_permission_state( &args.name, &args.url.as_deref(), - &resolved_path.as_deref().map(Path::new), + &path.as_deref().map(Path::new), )?; Ok(JsonOp::Sync(json!({ "state": perm.to_string() }))) } @@ -56,7 +47,7 @@ pub fn op_query_permission( pub fn op_revoke_permission( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: PermissionArgs = serde_json::from_value(args)?; let mut state = state.borrow_mut(); @@ -71,11 +62,11 @@ pub fn op_revoke_permission( "hrtime" => permissions.allow_hrtime.revoke(), _ => {} }; - let resolved_path = args.path.as_deref().map(resolve_path); + let path = args.path.as_deref(); let perm = permissions.get_permission_state( &args.name, &args.url.as_deref(), - &resolved_path.as_deref().map(Path::new), + &path.as_deref().map(Path::new), )?; Ok(JsonOp::Sync(json!({ "state": perm.to_string() }))) } @@ -83,20 +74,16 @@ pub fn op_revoke_permission( pub fn op_request_permission( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: PermissionArgs = serde_json::from_value(args)?; let mut state = state.borrow_mut(); let permissions = &mut state.permissions; - let resolved_path = args.path.as_deref().map(resolve_path); + let path = args.path.as_deref(); let perm = match args.name.as_ref() { "run" => Ok(permissions.request_run()), - "read" => { - Ok(permissions.request_read(&resolved_path.as_deref().map(Path::new))) - } - "write" => { - Ok(permissions.request_write(&resolved_path.as_deref().map(Path::new))) - } + "read" => Ok(permissions.request_read(&path.as_deref().map(Path::new))), + "write" => Ok(permissions.request_write(&path.as_deref().map(Path::new))), "net" => permissions.request_net(&args.url.as_deref()), "env" => Ok(permissions.request_env()), "plugin" => Ok(permissions.request_plugin()), diff --git a/cli/ops/plugin.rs b/cli/ops/plugin.rs index cabb3329db72cd..775178f1eda612 100644 --- a/cli/ops/plugin.rs +++ b/cli/ops/plugin.rs @@ -1,5 +1,4 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -use crate::fs::resolve_from_cwd; use crate::op_error::OpError; use crate::ops::dispatch_json::Deserialize; use crate::ops::dispatch_json::JsonOp; @@ -8,13 +7,14 @@ use crate::ops::json_op; use crate::state::State; use deno_core::plugin_api; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::Op; use deno_core::OpAsyncFuture; use deno_core::OpId; use deno_core::ZeroCopyBuf; use dlopen::symbor::Library; use futures::prelude::*; -use std::path::Path; +use std::path::PathBuf; use std::pin::Pin; use std::rc::Rc; use std::task::Context; @@ -34,14 +34,14 @@ struct OpenPluginArgs { } pub fn op_open_plugin( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.openPlugin"); let args: OpenPluginArgs = serde_json::from_value(args).unwrap(); - let filename = resolve_from_cwd(Path::new(&args.filename))?; + let filename = PathBuf::from(&args.filename); state.check_plugin(&filename)?; @@ -51,7 +51,7 @@ pub fn op_open_plugin( .map_err(OpError::from)?; let plugin_resource = PluginResource::new(&plugin_lib); - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let rid = resource_table.add("plugin", Box::new(plugin_resource)); let plugin_resource = resource_table.get::(rid).unwrap(); @@ -63,7 +63,7 @@ pub fn op_open_plugin( .unwrap(); drop(resource_table); - let mut interface = PluginInterface::new(isolate, &plugin_lib); + let mut interface = PluginInterface::new(isolate_state, &plugin_lib); deno_plugin_init(&mut interface); Ok(JsonOp::Sync(json!(rid))) @@ -80,14 +80,17 @@ impl PluginResource { } struct PluginInterface<'a> { - isolate: &'a mut CoreIsolate, + isolate_state: &'a mut CoreIsolateState, plugin_lib: &'a Rc, } impl<'a> PluginInterface<'a> { - fn new(isolate: &'a mut CoreIsolate, plugin_lib: &'a Rc) -> Self { + fn new( + isolate_state: &'a mut CoreIsolateState, + plugin_lib: &'a Rc, + ) -> Self { Self { - isolate, + isolate_state, plugin_lib, } } @@ -105,10 +108,10 @@ impl<'a> plugin_api::Interface for PluginInterface<'a> { dispatch_op_fn: plugin_api::DispatchOpFn, ) -> OpId { let plugin_lib = self.plugin_lib.clone(); - self.isolate.op_registry.register( + self.isolate_state.op_registry.register( name, - move |isolate, control, zero_copy| { - let mut interface = PluginInterface::new(isolate, &plugin_lib); + move |isolate_state, control, zero_copy| { + let mut interface = PluginInterface::new(isolate_state, &plugin_lib); let op = dispatch_op_fn(&mut interface, control, zero_copy); match op { sync_op @ Op::Sync(..) => sync_op, diff --git a/cli/ops/process.rs b/cli/ops/process.rs index 125aa136b44da4..78d9313c0ceafd 100644 --- a/cli/ops/process.rs +++ b/cli/ops/process.rs @@ -5,6 +5,7 @@ use crate::op_error::OpError; use crate::signal::kill; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ResourceTable; use deno_core::ZeroCopyBuf; use futures::future::poll_fn; @@ -60,15 +61,15 @@ struct ChildResource { } fn op_run( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let run_args: RunArgs = serde_json::from_value(args)?; state.check_run()?; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let args = run_args.cmd; let env = run_args.env; @@ -174,16 +175,16 @@ struct RunStatusArgs { } fn op_run_status( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: RunStatusArgs = serde_json::from_value(args)?; let rid = args.rid as u32; state.check_run()?; - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); let future = async move { let run_status = poll_fn(|cx| { @@ -227,7 +228,7 @@ struct KillArgs { fn op_kill( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.kill"); state.check_run()?; diff --git a/cli/ops/random.rs b/cli/ops/random.rs index 874887cdb40863..b29c761b860562 100644 --- a/cli/ops/random.rs +++ b/cli/ops/random.rs @@ -17,15 +17,15 @@ pub fn init(i: &mut CoreIsolate, s: &State) { fn op_get_random_values( state: &State, _args: Value, - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> Result { - assert!(zero_copy.is_some()); + assert_eq!(zero_copy.len(), 1); if let Some(ref mut seeded_rng) = state.borrow_mut().seeded_rng { - seeded_rng.fill(&mut zero_copy.unwrap()[..]); + seeded_rng.fill(&mut *zero_copy[0]); } else { let mut rng = thread_rng(); - rng.fill(&mut zero_copy.unwrap()[..]); + rng.fill(&mut *zero_copy[0]); } Ok(JsonOp::Sync(json!({}))) diff --git a/cli/ops/repl.rs b/cli/ops/repl.rs index 7dc0d0263da07f..b8fd7ab8b3c91a 100644 --- a/cli/ops/repl.rs +++ b/cli/ops/repl.rs @@ -5,6 +5,7 @@ use crate::repl; use crate::repl::Repl; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ZeroCopyBuf; use std::sync::Arc; use std::sync::Mutex; @@ -23,10 +24,10 @@ struct ReplStartArgs { } fn op_repl_start( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ReplStartArgs = serde_json::from_value(args)?; debug!("op_repl_start {}", args.history_file); @@ -34,7 +35,7 @@ fn op_repl_start( repl::history_path(&state.borrow().global_state.dir, &args.history_file); let repl = repl::Repl::new(history_path); let resource = ReplResource(Arc::new(Mutex::new(repl))); - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let rid = resource_table.add("repl", Box::new(resource)); Ok(JsonOp::Sync(json!(rid))) } @@ -46,16 +47,16 @@ struct ReplReadlineArgs { } fn op_repl_readline( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ReplReadlineArgs = serde_json::from_value(args)?; let rid = args.rid as u32; let prompt = args.prompt; debug!("op_repl_readline {} {}", rid, prompt); - let resource_table = isolate.resource_table.borrow(); + let resource_table = isolate_state.resource_table.borrow(); let resource = resource_table .get::(rid) .ok_or_else(OpError::bad_resource_id)?; diff --git a/cli/ops/resources.rs b/cli/ops/resources.rs index 1aa8dd4dc2cce7..a66a661700ee90 100644 --- a/cli/ops/resources.rs +++ b/cli/ops/resources.rs @@ -3,6 +3,7 @@ use super::dispatch_json::{Deserialize, JsonOp, Value}; use crate::op_error::OpError; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ZeroCopyBuf; pub fn init(i: &mut CoreIsolate, s: &State) { @@ -11,28 +12,28 @@ pub fn init(i: &mut CoreIsolate, s: &State) { } fn op_resources( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { - let serialized_resources = isolate.resource_table.borrow().entries(); + let serialized_resources = isolate_state.resource_table.borrow().entries(); Ok(JsonOp::Sync(json!(serialized_resources))) } /// op_close removes a resource from the resource table. fn op_close( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { #[derive(Deserialize)] struct CloseArgs { rid: i32, } let args: CloseArgs = serde_json::from_value(args)?; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); resource_table .close(args.rid as u32) .ok_or_else(OpError::bad_resource_id)?; diff --git a/cli/ops/runtime.rs b/cli/ops/runtime.rs index e96a669d048a42..a85ce4011d5bc2 100644 --- a/cli/ops/runtime.rs +++ b/cli/ops/runtime.rs @@ -17,7 +17,7 @@ pub fn init(i: &mut CoreIsolate, s: &State) { fn op_start( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let state = state.borrow(); let gs = &state.global_state; @@ -42,7 +42,7 @@ fn op_start( fn op_metrics( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let state = state.borrow(); let m = &state.metrics; diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs index f3b741861a8668..97102ef8196492 100644 --- a/cli/ops/runtime_compiler.rs +++ b/cli/ops/runtime_compiler.rs @@ -26,7 +26,7 @@ struct CompileArgs { fn op_compile( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.compile"); let args: CompileArgs = serde_json::from_value(args)?; @@ -57,7 +57,7 @@ struct TranspileArgs { fn op_transpile( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.transpile"); let args: TranspileArgs = serde_json::from_value(args)?; diff --git a/cli/ops/signal.rs b/cli/ops/signal.rs index ef652bc67153cd..6457ac42e484e7 100644 --- a/cli/ops/signal.rs +++ b/cli/ops/signal.rs @@ -3,6 +3,7 @@ use super::dispatch_json::{JsonOp, Value}; use crate::op_error::OpError; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ZeroCopyBuf; #[cfg(unix)] @@ -39,14 +40,14 @@ struct SignalArgs { #[cfg(unix)] fn op_signal_bind( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.signal"); let args: BindSignalArgs = serde_json::from_value(args)?; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let rid = resource_table.add( "signal", Box::new(SignalStreamResource( @@ -61,15 +62,15 @@ fn op_signal_bind( #[cfg(unix)] fn op_signal_poll( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.signal"); let args: SignalArgs = serde_json::from_value(args)?; let rid = args.rid as u32; - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); let future = poll_fn(move |cx| { let mut resource_table = resource_table.borrow_mut(); @@ -88,15 +89,15 @@ fn op_signal_poll( #[cfg(unix)] pub fn op_signal_unbind( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.signal"); let args: SignalArgs = serde_json::from_value(args)?; let rid = args.rid as u32; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let resource = resource_table.get::(rid); if let Some(signal) = resource { if let Some(waker) = &signal.1 { @@ -113,30 +114,30 @@ pub fn op_signal_unbind( #[cfg(not(unix))] pub fn op_signal_bind( - _isolate: &mut CoreIsolate, + _isolate_state: &mut CoreIsolateState, _state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { unimplemented!(); } #[cfg(not(unix))] fn op_signal_unbind( - _isolate: &mut CoreIsolate, + _isolate_state: &mut CoreIsolateState, _state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { unimplemented!(); } #[cfg(not(unix))] fn op_signal_poll( - _isolate: &mut CoreIsolate, + _isolate_state: &mut CoreIsolateState, _state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { unimplemented!(); } diff --git a/cli/ops/timers.rs b/cli/ops/timers.rs index e5bc461d03e887..044c5ea4afa841 100644 --- a/cli/ops/timers.rs +++ b/cli/ops/timers.rs @@ -20,7 +20,7 @@ pub fn init(i: &mut CoreIsolate, s: &State) { fn op_global_timer_stop( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let mut state = state.borrow_mut(); state.global_timer.cancel(); @@ -35,7 +35,7 @@ struct GlobalTimerArgs { fn op_global_timer( state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: GlobalTimerArgs = serde_json::from_value(args)?; let val = args.timeout; @@ -57,7 +57,7 @@ fn op_global_timer( fn op_now( state: &State, _args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let state = state.borrow(); let seconds = state.start_time.elapsed().as_secs(); diff --git a/cli/ops/tls.rs b/cli/ops/tls.rs index 5cbdff3007e33f..3e22c71ea67253 100644 --- a/cli/ops/tls.rs +++ b/cli/ops/tls.rs @@ -5,6 +5,7 @@ use crate::op_error::OpError; use crate::resolve_addr::resolve_addr; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ZeroCopyBuf; use futures::future::poll_fn; use futures::future::FutureExt; @@ -53,16 +54,16 @@ struct StartTLSArgs { } pub fn op_start_tls( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.startTls"); let args: StartTLSArgs = serde_json::from_value(args)?; let rid = args.rid as u32; let cert_file = args.cert_file.clone(); - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); let mut domain = args.hostname; if domain.is_empty() { @@ -132,14 +133,14 @@ pub fn op_start_tls( } pub fn op_connect_tls( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ConnectTLSArgs = serde_json::from_value(args)?; let cert_file = args.cert_file.clone(); - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); state.check_net(&args.hostname, args.port)?; if let Some(path) = cert_file.clone() { state.check_read(Path::new(&path))?; @@ -306,10 +307,10 @@ struct ListenTlsArgs { } fn op_listen_tls( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: ListenTlsArgs = serde_json::from_value(args)?; assert_eq!(args.transport, "tcp"); @@ -337,7 +338,7 @@ fn op_listen_tls( local_addr, }; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let rid = resource_table.add("tlsListener", Box::new(tls_listener_resource)); Ok(JsonOp::Sync(json!({ @@ -356,14 +357,14 @@ struct AcceptTlsArgs { } fn op_accept_tls( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: AcceptTlsArgs = serde_json::from_value(args)?; let rid = args.rid as u32; - let resource_table = isolate.resource_table.clone(); + let resource_table = isolate_state.resource_table.clone(); let op = async move { let accept_fut = poll_fn(|cx| { let mut resource_table = resource_table.borrow_mut(); diff --git a/cli/ops/tty.rs b/cli/ops/tty.rs index cf7f62ed889a50..ee357ed68ab2c2 100644 --- a/cli/ops/tty.rs +++ b/cli/ops/tty.rs @@ -4,6 +4,7 @@ use super::io::{StreamResource, StreamResourceHolder}; use crate::op_error::OpError; use crate::state::State; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ZeroCopyBuf; #[cfg(unix)] use nix::sys::termios; @@ -46,10 +47,10 @@ struct SetRawArgs { } pub fn op_set_raw( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { state.check_unstable("Deno.setRaw"); let args: SetRawArgs = serde_json::from_value(args)?; @@ -67,7 +68,7 @@ pub fn op_set_raw( use winapi::shared::minwindef::FALSE; use winapi::um::{consoleapi, handleapi}; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let resource_holder = resource_table.get_mut::(rid); if resource_holder.is_none() { return Err(OpError::bad_resource_id()); @@ -133,7 +134,7 @@ pub fn op_set_raw( { use std::os::unix::io::AsRawFd; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let resource_holder = resource_table.get_mut::(rid); if resource_holder.is_none() { return Err(OpError::bad_resource_id()); @@ -215,15 +216,15 @@ struct IsattyArgs { } pub fn op_isatty( - isolate: &mut CoreIsolate, + isolate_state: &mut CoreIsolateState, _state: &State, args: Value, - _zero_copy: Option, + _zero_copy: &mut [ZeroCopyBuf], ) -> Result { let args: IsattyArgs = serde_json::from_value(args)?; let rid = args.rid; - let mut resource_table = isolate.resource_table.borrow_mut(); + let mut resource_table = isolate_state.resource_table.borrow_mut(); let isatty: bool = std_file_resource(&mut resource_table, rid as u32, move |r| match r { Ok(std_file) => { diff --git a/cli/ops/web_worker.rs b/cli/ops/web_worker.rs index e95eb8fe110fc0..553278b0741e1c 100644 --- a/cli/ops/web_worker.rs +++ b/cli/ops/web_worker.rs @@ -6,6 +6,7 @@ use crate::state::State; use crate::web_worker::WebWorkerHandle; use crate::worker::WorkerEvent; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ZeroCopyBuf; use futures::channel::mpsc; use std::convert::From; @@ -13,17 +14,21 @@ use std::convert::From; pub fn web_worker_op( sender: mpsc::Sender, dispatcher: D, -) -> impl Fn(&mut CoreIsolate, Value, Option) -> Result +) -> impl Fn( + &mut CoreIsolateState, + Value, + &mut [ZeroCopyBuf], +) -> Result where D: Fn( &mpsc::Sender, Value, - Option, + &mut [ZeroCopyBuf], ) -> Result, { - move |_isolate: &mut CoreIsolate, + move |_isolate_state: &mut CoreIsolateState, args: Value, - zero_copy: Option| + zero_copy: &mut [ZeroCopyBuf]| -> Result { dispatcher(&sender, args, zero_copy) } } @@ -31,18 +36,22 @@ pub fn web_worker_op2( handle: WebWorkerHandle, sender: mpsc::Sender, dispatcher: D, -) -> impl Fn(&mut CoreIsolate, Value, Option) -> Result +) -> impl Fn( + &mut CoreIsolateState, + Value, + &mut [ZeroCopyBuf], +) -> Result where D: Fn( WebWorkerHandle, &mpsc::Sender, Value, - Option, + &mut [ZeroCopyBuf], ) -> Result, { - move |_isolate: &mut CoreIsolate, + move |_isolate_state: &mut CoreIsolateState, args: Value, - zero_copy: Option| + zero_copy: &mut [ZeroCopyBuf]| -> Result { dispatcher(handle.clone(), &sender, args, zero_copy) } @@ -75,9 +84,10 @@ pub fn init( fn op_worker_post_message( sender: &mpsc::Sender, _args: Value, - data: Option, + data: &mut [ZeroCopyBuf], ) -> Result { - let d = Vec::from(data.unwrap().as_ref()).into_boxed_slice(); + assert_eq!(data.len(), 1, "Invalid number of arguments"); + let d = Vec::from(&*data[0]).into_boxed_slice(); let mut sender = sender.clone(); sender .try_send(WorkerEvent::Message(d)) @@ -90,7 +100,7 @@ fn op_worker_close( handle: WebWorkerHandle, sender: &mpsc::Sender, _args: Value, - _data: Option, + _data: &mut [ZeroCopyBuf], ) -> Result { let mut sender = sender.clone(); // Notify parent that we're finished diff --git a/cli/ops/worker_host.rs b/cli/ops/worker_host.rs index d1b4bb80f1f1c1..0ed700431afaf0 100644 --- a/cli/ops/worker_host.rs +++ b/cli/ops/worker_host.rs @@ -54,7 +54,9 @@ fn create_web_worker( ); if has_deno_namespace { - let mut resource_table = worker.resource_table.borrow_mut(); + let state_rc = CoreIsolate::state(&worker.isolate); + let state = state_rc.borrow(); + let mut resource_table = state.resource_table.borrow_mut(); let (stdin, stdout, stderr) = get_stdio(); resource_table.add("stdin", Box::new(stdin)); resource_table.add("stdout", Box::new(stdout)); @@ -172,7 +174,7 @@ struct CreateWorkerArgs { fn op_create_worker( state: &State, args: Value, - _data: Option, + _data: &mut [ZeroCopyBuf], ) -> Result { let args: CreateWorkerArgs = serde_json::from_value(args)?; @@ -228,7 +230,7 @@ struct WorkerArgs { fn op_host_terminate_worker( state: &State, args: Value, - _data: Option, + _data: &mut [ZeroCopyBuf], ) -> Result { let args: WorkerArgs = serde_json::from_value(args)?; let id = args.id as u32; @@ -294,7 +296,7 @@ fn serialize_worker_event(event: WorkerEvent) -> Value { fn op_host_get_message( state: &State, args: Value, - _data: Option, + _data: &mut [ZeroCopyBuf], ) -> Result { let args: WorkerArgs = serde_json::from_value(args)?; let id = args.id as u32; @@ -343,11 +345,12 @@ fn op_host_get_message( fn op_host_post_message( state: &State, args: Value, - data: Option, + data: &mut [ZeroCopyBuf], ) -> Result { + assert_eq!(data.len(), 1, "Invalid number of arguments"); let args: WorkerArgs = serde_json::from_value(args)?; let id = args.id as u32; - let msg = Vec::from(data.unwrap().as_ref()).into_boxed_slice(); + let msg = Vec::from(&*data[0]).into_boxed_slice(); debug!("post message to worker {}", id); let state = state.borrow(); diff --git a/cli/permissions.rs b/cli/permissions.rs index bc4d0844e0ec63..45c36591cdf014 100644 --- a/cli/permissions.rs +++ b/cli/permissions.rs @@ -1,8 +1,12 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. use crate::colors; use crate::flags::Flags; +use crate::fs::resolve_from_cwd; use crate::op_error::OpError; +use serde::de; +use serde::Deserialize; use std::collections::HashSet; +use std::env::current_dir; use std::fmt; #[cfg(not(test))] use std::io; @@ -57,6 +61,18 @@ impl PermissionState { } *self } + + pub fn fork(self, value: bool) -> Result { + if value && self == PermissionState::Deny { + Err(OpError::permission_denied( + "Arguments escalate parent permissions.".to_string(), + )) + } else if value { + Ok(PermissionState::Allow) + } else { + Ok(PermissionState::Deny) + } + } } impl From for PermissionState { @@ -96,35 +112,71 @@ impl Default for PermissionState { } } -#[derive(Clone, Debug, Default)] +struct BoolPermVisitor; + +fn deserialize_permission_state<'de, D>( + d: D, +) -> Result +where + D: de::Deserializer<'de>, +{ + impl<'de> de::Visitor<'de> for BoolPermVisitor { + type Value = PermissionState; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a boolean value") + } + + fn visit_bool(self, value: bool) -> Result + where + E: de::Error, + { + if value { + Ok(PermissionState::Allow) + } else { + Ok(PermissionState::Deny) + } + } + } + d.deserialize_bool(BoolPermVisitor) +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] pub struct Permissions { // Keep in sync with cli/js/permissions.ts + #[serde(deserialize_with = "deserialize_permission_state")] pub allow_read: PermissionState, pub read_whitelist: HashSet, + #[serde(deserialize_with = "deserialize_permission_state")] pub allow_write: PermissionState, pub write_whitelist: HashSet, + #[serde(deserialize_with = "deserialize_permission_state")] pub allow_net: PermissionState, pub net_whitelist: HashSet, + #[serde(deserialize_with = "deserialize_permission_state")] pub allow_env: PermissionState, + #[serde(deserialize_with = "deserialize_permission_state")] pub allow_run: PermissionState, + #[serde(deserialize_with = "deserialize_permission_state")] pub allow_plugin: PermissionState, + #[serde(deserialize_with = "deserialize_permission_state")] pub allow_hrtime: PermissionState, } +fn resolve_fs_whitelist(whitelist: &[PathBuf]) -> HashSet { + whitelist + .iter() + .map(|raw_path| resolve_from_cwd(Path::new(&raw_path)).unwrap()) + .collect() +} + impl Permissions { pub fn from_flags(flags: &Flags) -> Self { - // assert each whitelist path is absolute, since the cwd may change. - for path in &flags.read_whitelist { - assert!(path.has_root()); - } - for path in &flags.write_whitelist { - assert!(path.has_root()); - } Self { allow_read: PermissionState::from(flags.allow_read), - read_whitelist: flags.read_whitelist.iter().cloned().collect(), + read_whitelist: resolve_fs_whitelist(&flags.read_whitelist), allow_write: PermissionState::from(flags.allow_write), - write_whitelist: flags.write_whitelist.iter().cloned().collect(), + write_whitelist: resolve_fs_whitelist(&flags.write_whitelist), allow_net: PermissionState::from(flags.allow_net), net_whitelist: flags.net_whitelist.iter().cloned().collect(), allow_env: PermissionState::from(flags.allow_env), @@ -134,6 +186,24 @@ impl Permissions { } } + /// Arbitrary helper. Resolves the path from CWD, and also gets a path that + /// can be displayed without leaking the CWD when not allowed. + fn resolved_and_display_path(&self, path: &Path) -> (PathBuf, PathBuf) { + let resolved_path = resolve_from_cwd(path).unwrap(); + let display_path = if path.is_absolute() { + path.to_path_buf() + } else { + match self + .get_state_read(&Some(¤t_dir().unwrap())) + .check("", "") + { + Ok(_) => resolved_path.clone(), + Err(_) => path.to_path_buf(), + } + }; + (resolved_path, display_path) + } + pub fn allow_all() -> Self { Self { allow_read: PermissionState::from(true), @@ -161,12 +231,26 @@ impl Permissions { } pub fn check_read(&self, path: &Path) -> Result<(), OpError> { - self.get_state_read(&Some(path)).check( - &format!("read access to \"{}\"", path.display()), + let (resolved_path, display_path) = self.resolved_and_display_path(path); + self.get_state_read(&Some(&resolved_path)).check( + &format!("read access to \"{}\"", display_path.display()), "--allow-read", ) } + /// As `check_read()`, but permission error messages will anonymize the path + /// by replacing it with the given `display`. + pub fn check_read_blind( + &self, + path: &Path, + display: &str, + ) -> Result<(), OpError> { + let resolved_path = resolve_from_cwd(path).unwrap(); + self + .get_state_read(&Some(&resolved_path)) + .check(&format!("read access to <{}>", display), "--allow-read") + } + fn get_state_write(&self, path: &Option<&Path>) -> PermissionState { if path.map_or(false, |f| check_path_white_list(f, &self.write_whitelist)) { return PermissionState::Allow; @@ -175,8 +259,9 @@ impl Permissions { } pub fn check_write(&self, path: &Path) -> Result<(), OpError> { - self.get_state_write(&Some(path)).check( - &format!("write access to \"{}\"", path.display()), + let (resolved_path, display_path) = self.resolved_and_display_path(path); + self.get_state_write(&Some(&resolved_path)).check( + &format!("write access to \"{}\"", display_path.display()), "--allow-write", ) } @@ -226,8 +311,9 @@ impl Permissions { } pub fn check_plugin(&self, path: &Path) -> Result<(), OpError> { + let (_, display_path) = self.resolved_and_display_path(path); self.allow_plugin.check( - &format!("access to open a plugin: {}", path.display()), + &format!("access to open a plugin: {}", display_path.display()), "--allow-plugin", ) } @@ -239,26 +325,34 @@ impl Permissions { } pub fn request_read(&mut self, path: &Option<&Path>) -> PermissionState { - if path.map_or(false, |f| check_path_white_list(f, &self.read_whitelist)) { - return PermissionState::Allow; + let paths = path.map(|p| self.resolved_and_display_path(p)); + if let Some((p, _)) = paths.as_ref() { + if check_path_white_list(&p, &self.read_whitelist) { + return PermissionState::Allow; + } }; - self.allow_read.request(&match path { + self.allow_read.request(&match paths { None => "Deno requests read access".to_string(), - Some(path) => { - format!("Deno requests read access to \"{}\"", path.display()) - } + Some((_, display_path)) => format!( + "Deno requests read access to \"{}\"", + display_path.display() + ), }) } pub fn request_write(&mut self, path: &Option<&Path>) -> PermissionState { - if path.map_or(false, |f| check_path_white_list(f, &self.write_whitelist)) { - return PermissionState::Allow; + let paths = path.map(|p| self.resolved_and_display_path(p)); + if let Some((p, _)) = paths.as_ref() { + if check_path_white_list(&p, &self.write_whitelist) { + return PermissionState::Allow; + } }; - self.allow_write.request(&match path { + self.allow_write.request(&match paths { None => "Deno requests write access".to_string(), - Some(path) => { - format!("Deno requests write access to \"{}\"", path.display()) - } + Some((_, display_path)) => format!( + "Deno requests write access to \"{}\"", + display_path.display() + ), }) } @@ -297,10 +391,12 @@ impl Permissions { url: &Option<&str>, path: &Option<&Path>, ) -> Result { + let path = path.map(|p| resolve_from_cwd(p).unwrap()); + let path = path.as_deref(); match name { "run" => Ok(self.allow_run), - "read" => Ok(self.get_state_read(path)), - "write" => Ok(self.get_state_write(path)), + "read" => Ok(self.get_state_read(&path)), + "write" => Ok(self.get_state_write(&path)), "net" => self.get_state_net_url(url), "env" => Ok(self.allow_env), "plugin" => Ok(self.allow_plugin), @@ -308,6 +404,58 @@ impl Permissions { n => Err(OpError::other(format!("No such permission name: {}", n))), } } + + #[allow(clippy::too_many_arguments)] + pub fn fork( + &self, + allow_read: bool, + read_whitelist: HashSet, + allow_write: bool, + write_whitelist: HashSet, + allow_net: bool, + net_whitelist: HashSet, + allow_env: bool, + allow_run: bool, + allow_plugin: bool, + allow_hrtime: bool, + ) -> Result { + let allow_read = self.allow_read.fork(allow_read)?; + let allow_write = self.allow_write.fork(allow_write)?; + let allow_net = self.allow_net.fork(allow_net)?; + let allow_env = self.allow_env.fork(allow_env)?; + let allow_run = self.allow_run.fork(allow_run)?; + let allow_plugin = self.allow_plugin.fork(allow_plugin)?; + let allow_hrtime = self.allow_hrtime.fork(allow_hrtime)?; + if !(read_whitelist.is_subset(&self.read_whitelist)) { + Err(OpError::permission_denied(format!( + "Arguments escalate parent permissions. Parent Permissions have only {:?} in `read_whitelist`", + self.read_whitelist + ))) + } else if !(write_whitelist.is_subset(&self.write_whitelist)) { + Err(OpError::permission_denied(format!( + "Arguments escalate parent permissions. Parent Permissions have only {:?} in `write_whitelist`", + self.write_whitelist + ))) + } else if !(net_whitelist.is_subset(&self.net_whitelist)) { + Err(OpError::permission_denied(format!( + "Arguments escalate parent permissions. Parent Permissions have only {:?} in `net_whitelist`", + self.net_whitelist + ))) + } else { + Ok(Permissions { + allow_read, + read_whitelist, + allow_write, + write_whitelist, + allow_net, + net_whitelist, + allow_env, + allow_run, + allow_plugin, + allow_hrtime, + }) + } + } } /// Shows the permission prompt and returns the answer according to the user input. @@ -448,6 +596,14 @@ mod tests { assert!(perms.check_read(Path::new("/b/c/sub/path")).is_ok()); assert!(perms.check_write(Path::new("/b/c/sub/path")).is_ok()); + // Sub path within /b/c, needs normalizing + assert!(perms + .check_read(Path::new("/b/c/sub/path/../path/.")) + .is_ok()); + assert!(perms + .check_write(Path::new("/b/c/sub/path/../path/.")) + .is_ok()); + // Inside of /b but outside of /b/c assert!(perms.check_read(Path::new("/b/e")).is_err()); assert!(perms.check_write(Path::new("/b/e")).is_err()); @@ -735,4 +891,104 @@ mod tests { assert_eq!(perms1.request_hrtime(), PermissionState::Deny); drop(guard); } + + #[test] + fn test_deserialize_perms() { + let json_perms = r#" + { + "allow_read": true, + "read_whitelist": [], + "allow_write": true, + "write_whitelist": [], + "allow_net": true, + "net_whitelist": [], + "allow_env": true, + "allow_run": true, + "allow_plugin": true, + "allow_hrtime": true + } + "#; + let perms0 = Permissions { + allow_read: PermissionState::Allow, + allow_write: PermissionState::Allow, + allow_net: PermissionState::Allow, + allow_hrtime: PermissionState::Allow, + allow_env: PermissionState::Allow, + allow_plugin: PermissionState::Allow, + allow_run: PermissionState::Allow, + read_whitelist: HashSet::new(), + write_whitelist: HashSet::new(), + net_whitelist: HashSet::new(), + }; + let deserialized_perms: Permissions = + serde_json::from_str(json_perms).unwrap(); + assert_eq!(perms0, deserialized_perms); + } + + #[test] + fn test_fork() { + let guard = PERMISSION_PROMPT_GUARD.lock().unwrap(); + let perms0 = Permissions::from_flags(&Flags { + ..Default::default() + }); + set_prompt_result(true); + assert_eq!( + perms0 + .fork( + true, + HashSet::new(), + true, + HashSet::new(), + true, + HashSet::new(), + true, + true, + false, + false, + ) + .expect("Testing expect"), + Permissions { + allow_read: PermissionState::Allow, + read_whitelist: HashSet::new(), + allow_write: PermissionState::Allow, + write_whitelist: HashSet::new(), + allow_net: PermissionState::Allow, + net_whitelist: HashSet::new(), + allow_env: PermissionState::Allow, + allow_run: PermissionState::Allow, + allow_plugin: PermissionState::Deny, + allow_hrtime: PermissionState::Deny, + } + ); + set_prompt_result(false); + assert_eq!( + perms0 + .fork( + true, + HashSet::new(), + true, + HashSet::new(), + true, + HashSet::new(), + true, + true, + false, + false, + ) + .expect("Testing expect"), + Permissions { + allow_read: PermissionState::Allow, + read_whitelist: HashSet::new(), + allow_write: PermissionState::Allow, + write_whitelist: HashSet::new(), + allow_net: PermissionState::Allow, + net_whitelist: HashSet::new(), + allow_env: PermissionState::Allow, + allow_run: PermissionState::Allow, + allow_plugin: PermissionState::Deny, + allow_hrtime: PermissionState::Deny, + } + ); + drop(guard); + } } diff --git a/cli/source_maps.rs b/cli/source_maps.rs index 90780e042ed702..d87147dbe61a9d 100644 --- a/cli/source_maps.rs +++ b/cli/source_maps.rs @@ -18,17 +18,6 @@ pub trait SourceMapGetter { /// find a SourceMap. pub type CachedMaps = HashMap>; -// The bundle does not get built for 'cargo check', so we don't embed the -// bundle source map. The built in source map is the source map for the main -// JavaScript bundle which is then used to create the snapshot. Runtime stack -// traces can contain positions within the bundle which we will map to the -// original Deno TypeScript code. -#[cfg(feature = "check-only")] -fn builtin_source_map(_: &str) -> Option> { - None -} - -#[cfg(not(feature = "check-only"))] fn builtin_source_map(file_name: &str) -> Option> { if file_name.ends_with("CLI_SNAPSHOT.js") { Some(crate::js::CLI_SNAPSHOT_MAP.to_vec()) diff --git a/cli/startup_data.rs b/cli/startup_data.rs index 517165d110c5f9..86bae2f47153b9 100644 --- a/cli/startup_data.rs +++ b/cli/startup_data.rs @@ -1,7 +1,4 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -#[cfg(feature = "no-snapshot-init")] -use deno_core::Script; - use crate::js::CLI_SNAPSHOT; use crate::js::COMPILER_SNAPSHOT; use deno_core::Snapshot; @@ -10,13 +7,9 @@ use deno_core::StartupData; #[cfg(feature = "no-snapshot-init")] pub fn deno_isolate_init() -> StartupData<'static> { debug!("Deno isolate init without snapshots."); - #[cfg(not(feature = "check-only"))] let source = include_str!(concat!(env!("GN_OUT_DIR"), "/gen/cli/bundle/main.js")); - #[cfg(feature = "check-only")] - let source = ""; - - StartupData::Script(Script { + StartupData::Script(deno_core::Script { filename: "gen/cli/bundle/main.js", source, }) @@ -25,24 +18,16 @@ pub fn deno_isolate_init() -> StartupData<'static> { #[cfg(not(feature = "no-snapshot-init"))] pub fn deno_isolate_init() -> StartupData<'static> { debug!("Deno isolate init with snapshots."); - #[cfg(not(feature = "check-only"))] let data = CLI_SNAPSHOT; - #[cfg(feature = "check-only")] - let data = b""; - StartupData::Snapshot(Snapshot::Static(data)) } #[cfg(feature = "no-snapshot-init")] pub fn compiler_isolate_init() -> StartupData<'static> { debug!("Compiler isolate init without snapshots."); - #[cfg(not(feature = "check-only"))] let source = include_str!(concat!(env!("GN_OUT_DIR"), "/gen/cli/bundle/compiler.js")); - #[cfg(feature = "check-only")] - let source = ""; - - StartupData::Script(Script { + StartupData::Script(deno_core::Script { filename: "gen/cli/bundle/compiler.js", source, }) @@ -51,10 +36,6 @@ pub fn compiler_isolate_init() -> StartupData<'static> { #[cfg(not(feature = "no-snapshot-init"))] pub fn compiler_isolate_init() -> StartupData<'static> { debug!("Deno isolate init with snapshots."); - #[cfg(not(feature = "check-only"))] let data = COMPILER_SNAPSHOT; - #[cfg(feature = "check-only")] - let data = b""; - StartupData::Snapshot(Snapshot::Static(data)) } diff --git a/cli/state.rs b/cli/state.rs index 46fbdfd0b27191..c94f927880224f 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -3,7 +3,6 @@ use crate::file_fetcher::SourceFileFetcher; use crate::global_state::GlobalState; use crate::global_timer::GlobalTimer; use crate::import_map::ImportMap; -use crate::inspector::DenoInspector; use crate::metrics::Metrics; use crate::op_error::OpError; use crate::ops::JsonOp; @@ -12,7 +11,6 @@ use crate::permissions::Permissions; use crate::tsc::TargetLib; use crate::web_worker::WebWorkerHandle; use deno_core::Buf; -use deno_core::CoreIsolate; use deno_core::ErrBox; use deno_core::ModuleLoadId; use deno_core::ModuleLoader; @@ -61,16 +59,15 @@ pub struct StateInner { pub target_lib: TargetLib, pub is_main: bool, pub is_internal: bool, - pub inspector: Option>, } impl State { pub fn stateful_json_op( &self, dispatcher: D, - ) -> impl Fn(&mut deno_core::CoreIsolate, &[u8], Option) -> Op + ) -> impl Fn(&mut deno_core::CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op where - D: Fn(&State, Value, Option) -> Result, + D: Fn(&State, Value, &mut [ZeroCopyBuf]) -> Result, { use crate::ops::json_op; self.core_op(json_op(self.stateful_op(dispatcher))) @@ -79,13 +76,13 @@ impl State { pub fn stateful_json_op2( &self, dispatcher: D, - ) -> impl Fn(&mut deno_core::CoreIsolate, &[u8], Option) -> Op + ) -> impl Fn(&mut deno_core::CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op where D: Fn( - &mut deno_core::CoreIsolate, + &mut deno_core::CoreIsolateState, &State, Value, - Option, + &mut [ZeroCopyBuf], ) -> Result, { use crate::ops::json_op; @@ -98,21 +95,21 @@ impl State { pub fn core_op( &self, dispatcher: D, - ) -> impl Fn(&mut deno_core::CoreIsolate, &[u8], Option) -> Op + ) -> impl Fn(&mut deno_core::CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op where - D: Fn(&mut deno_core::CoreIsolate, &[u8], Option) -> Op, + D: Fn(&mut deno_core::CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op, { let state = self.clone(); - move |isolate: &mut deno_core::CoreIsolate, + move |isolate_state: &mut deno_core::CoreIsolateState, control: &[u8], - zero_copy: Option| + zero_copy: &mut [ZeroCopyBuf]| -> Op { let bytes_sent_control = control.len() as u64; let bytes_sent_zero_copy = - zero_copy.as_ref().map(|b| b.len()).unwrap_or(0) as u64; + zero_copy.iter().map(|b| b.len()).sum::() as u64; - let op = dispatcher(isolate, control, zero_copy); + let op = dispatcher(isolate_state, control, zero_copy); match op { Op::Sync(buf) => { @@ -158,24 +155,24 @@ impl State { pub fn stateful_minimal_op2( &self, dispatcher: D, - ) -> impl Fn(&mut deno_core::CoreIsolate, &[u8], Option) -> Op + ) -> impl Fn(&mut deno_core::CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op where D: Fn( - &mut deno_core::CoreIsolate, + &mut deno_core::CoreIsolateState, &State, bool, i32, - Option, + &mut [ZeroCopyBuf], ) -> MinimalOp, { let state = self.clone(); self.core_op(crate::ops::minimal_op( - move |isolate: &mut deno_core::CoreIsolate, + move |isolate_state: &mut deno_core::CoreIsolateState, is_sync: bool, rid: i32, - zero_copy: Option| + zero_copy: &mut [ZeroCopyBuf]| -> MinimalOp { - dispatcher(isolate, &state, is_sync, rid, zero_copy) + dispatcher(isolate_state, &state, is_sync, rid, zero_copy) }, )) } @@ -189,17 +186,17 @@ impl State { &self, dispatcher: D, ) -> impl Fn( - &mut deno_core::CoreIsolate, + &mut deno_core::CoreIsolateState, Value, - Option, + &mut [ZeroCopyBuf], ) -> Result where - D: Fn(&State, Value, Option) -> Result, + D: Fn(&State, Value, &mut [ZeroCopyBuf]) -> Result, { let state = self.clone(); - move |_isolate: &mut deno_core::CoreIsolate, + move |_isolate_state: &mut deno_core::CoreIsolateState, args: Value, - zero_copy: Option| + zero_copy: &mut [ZeroCopyBuf]| -> Result { dispatcher(&state, args, zero_copy) } } @@ -207,24 +204,24 @@ impl State { &self, dispatcher: D, ) -> impl Fn( - &mut deno_core::CoreIsolate, + &mut deno_core::CoreIsolateState, Value, - Option, + &mut [ZeroCopyBuf], ) -> Result where D: Fn( - &mut deno_core::CoreIsolate, + &mut deno_core::CoreIsolateState, &State, Value, - Option, + &mut [ZeroCopyBuf], ) -> Result, { let state = self.clone(); - move |isolate: &mut deno_core::CoreIsolate, + move |isolate_state: &mut deno_core::CoreIsolateState, args: Value, - zero_copy: Option| + zero_copy: &mut [ZeroCopyBuf]| -> Result { - dispatcher(isolate, &state, args, zero_copy) + dispatcher(isolate_state, &state, args, zero_copy) } } @@ -271,76 +268,23 @@ impl ModuleLoader for State { Ok(module_specifier) } - /// Given an absolute url, load its source code. fn load( &self, module_specifier: &ModuleSpecifier, maybe_referrer: Option, - is_dyn_import: bool, + _is_dyn_import: bool, ) -> Pin> { - let module_specifier = module_specifier.clone(); - - // TODO(bartlomieju): this code is duplicated from module_graph. - // It should be removed when `prepare_load` will be used to load modules. - // Disallow http:// imports from modules loaded over https:// - if let Some(referrer) = maybe_referrer.as_ref() { - if let "https" = referrer.as_url().scheme() { - if let "http" = module_specifier.as_url().scheme() { - let e = OpError::permission_denied( - "Modules loaded over https:// are not allowed to import modules over http://".to_string() - ); - return async move { Err(e.into()) }.boxed_local(); - } - } - } - - if is_dyn_import { - if let Err(e) = self.check_dyn_import(&module_specifier) { - return async move { Err(e.into()) }.boxed_local(); - } - } else { - // Verify that remote file doesn't try to statically import local file. - if let Some(referrer) = maybe_referrer.as_ref() { - let referrer_url = referrer.as_url(); - match referrer_url.scheme() { - "http" | "https" => { - let specifier_url = module_specifier.as_url(); - match specifier_url.scheme() { - "http" | "https" => {} - _ => { - let e = OpError::permission_denied( - "Remote modules are not allowed to statically import local modules. Use dynamic import instead.".to_string() - ); - return async move { Err(e.into()) }.boxed_local(); - } - } - } - _ => {} - } - } - } - + let module_specifier = module_specifier.to_owned(); let mut state = self.borrow_mut(); // TODO(bartlomieju): incrementing resolve_count here has no sense... state.metrics.resolve_count += 1; let module_url_specified = module_specifier.to_string(); let global_state = state.global_state.clone(); - let target_lib = state.target_lib.clone(); - let permissions = if state.is_main { - Permissions::allow_all() - } else { - state.permissions.clone() - }; + // TODO(bartlomieju): `fetch_compiled_module` should take `load_id` param let fut = async move { let compiled_module = global_state - .fetch_compiled_module( - module_specifier, - maybe_referrer, - target_lib, - permissions, - is_dyn_import, - ) + .fetch_compiled_module(module_specifier, maybe_referrer) .await?; Ok(deno_core::ModuleSource { // Real module name, might be different from initial specifier @@ -357,22 +301,47 @@ impl ModuleLoader for State { fn prepare_load( &self, _load_id: ModuleLoadId, - _module_specifier: &ModuleSpecifier, - _maybe_referrer: Option, - _is_dyn_import: bool, + module_specifier: &ModuleSpecifier, + maybe_referrer: Option, + is_dyn_import: bool, ) -> Pin>>> { - // TODO(bartlomieju): - // 1. recursively: - // a) resolve specifier - // b) check permission if dynamic import - // c) fetch/download source code - // d) parse the source code and extract all import/exports (dependencies) - // e) add discovered deps and loop algorithm until no new dependencies - // are discovered - // 2. run through appropriate compiler giving it access only to - // discovered files - - async { Ok(()) }.boxed_local() + let module_specifier = module_specifier.clone(); + let state = self.borrow(); + let target_lib = state.target_lib.clone(); + let maybe_import_map = state.import_map.clone(); + // Only "main" module is loaded without permission check, + // ie. module that is associated with "is_main" state + // and is not a dynamic import. + let permissions = if state.is_main && !is_dyn_import { + Permissions::allow_all() + } else { + state.permissions.clone() + }; + let global_state = state.global_state.clone(); + // TODO(bartlomieju): I'm not sure if it's correct to ignore + // bad referrer - this is the case for `Deno.core.evalContext()` where + // `ref_str` is ``. + let maybe_referrer = if let Some(ref_str) = maybe_referrer { + ModuleSpecifier::resolve_url(&ref_str).ok() + } else { + None + }; + drop(state); + + // TODO(bartlomieju): `prepare_module_load` should take `load_id` param + async move { + global_state + .prepare_module_load( + module_specifier, + maybe_referrer, + target_lib, + permissions, + is_dyn_import, + maybe_import_map, + ) + .await + } + .boxed_local() } } @@ -382,19 +351,9 @@ impl State { global_state: GlobalState, shared_permissions: Option, main_module: ModuleSpecifier, + maybe_import_map: Option, is_internal: bool, ) -> Result { - let import_map: Option = - match global_state.flags.import_map_path.as_ref() { - None => None, - Some(file_path) => { - if !global_state.flags.unstable { - exit_unstable("--importmap") - } - Some(ImportMap::load(file_path)?) - } - }; - let seeded_rng = match global_state.flags.seed { Some(seed) => Some(StdRng::seed_from_u64(seed)), None => None, @@ -410,7 +369,7 @@ impl State { global_state, main_module, permissions, - import_map, + import_map: maybe_import_map, metrics: Metrics::default(), global_timer: GlobalTimer::new(), workers: HashMap::new(), @@ -420,7 +379,6 @@ impl State { target_lib: TargetLib::Main, is_main: true, is_internal, - inspector: None, })); Ok(Self(state)) @@ -457,7 +415,6 @@ impl State { target_lib: TargetLib::Worker, is_main: false, is_internal: false, - inspector: None, })); Ok(Self(state)) @@ -468,6 +425,17 @@ impl State { self.borrow().permissions.check_read(path) } + /// As `check_read()`, but permission error messages will anonymize the path + /// by replacing it with the given `display`. + #[inline] + pub fn check_read_blind( + &self, + path: &Path, + display: &str, + ) -> Result<(), OpError> { + self.borrow().permissions.check_read_blind(path, display) + } + #[inline] pub fn check_write(&self, path: &Path) -> Result<(), OpError> { self.borrow().permissions.check_write(path) @@ -526,37 +494,6 @@ impl State { } } - pub fn maybe_init_inspector(&self, isolate: &mut CoreIsolate) { - let mut state = self.borrow_mut(); - - if state.is_internal { - return; - }; - - let inspector_host = { - let global_state = &state.global_state; - match global_state - .flags - .inspect - .or(global_state.flags.inspect_brk) - { - Some(host) => host, - None => return, - } - }; - - let inspector = DenoInspector::new(isolate, inspector_host); - state.inspector.replace(inspector); - } - - #[inline] - pub fn should_inspector_break_on_first_statement(&self) -> bool { - let state = self.borrow(); - state.inspector.is_some() - && state.is_main - && state.global_state.flags.inspect_brk.is_some() - } - #[cfg(test)] pub fn mock(main_module: &str) -> State { let module_specifier = ModuleSpecifier::resolve_url_or_path(main_module) @@ -565,6 +502,7 @@ impl State { GlobalState::mock(vec!["deno".to_string()]), None, module_specifier, + None, false, ) .unwrap() diff --git a/cli/swc_util.rs b/cli/swc_util.rs index 465fb9769e763c..0d75877bf39aff 100644 --- a/cli/swc_util.rs +++ b/cli/swc_util.rs @@ -1,4 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::doc::Location; +use crate::msg::MediaType; use crate::swc_common; use crate::swc_common::comments::CommentKind; use crate::swc_common::comments::Comments; @@ -13,19 +15,42 @@ use crate::swc_common::SourceMap; use crate::swc_common::Span; use crate::swc_ecma_ast; use crate::swc_ecma_parser::lexer::Lexer; +use crate::swc_ecma_parser::EsConfig; use crate::swc_ecma_parser::JscTarget; use crate::swc_ecma_parser::Parser; use crate::swc_ecma_parser::Session; use crate::swc_ecma_parser::SourceFileInput; use crate::swc_ecma_parser::Syntax; use crate::swc_ecma_parser::TsConfig; -use swc_ecma_visit::Node; -use swc_ecma_visit::Visit; - use std::error::Error; use std::fmt; use std::sync::Arc; use std::sync::RwLock; +use swc_ecma_visit::Node; +use swc_ecma_visit::Visit; + +fn get_default_es_config() -> EsConfig { + let mut config = EsConfig::default(); + config.num_sep = true; + config.class_private_props = true; + config.class_private_methods = true; + config.class_props = true; + config.export_default_from = true; + config.export_namespace_from = true; + config.dynamic_import = true; + config.nullish_coalescing = true; + config.optional_chaining = true; + config.import_meta = true; + config.top_level_await = true; + config +} + +fn get_default_ts_config() -> TsConfig { + let mut ts_config = TsConfig::default(); + ts_config.dynamic_import = true; + ts_config.decorators = true; + ts_config +} #[derive(Clone, Debug)] pub struct SwcDiagnosticBuffer { @@ -126,6 +151,7 @@ impl AstParser { pub fn parse_module( &self, file_name: &str, + media_type: MediaType, source_code: &str, callback: F, ) -> R @@ -143,12 +169,21 @@ impl AstParser { handler: &self.handler, }; - // TODO(bartlomieju): lexer should be configurable by the caller - let mut ts_config = TsConfig::default(); - ts_config.dynamic_import = true; - ts_config.decorators = true; - ts_config.tsx = true; - let syntax = Syntax::Typescript(ts_config); + let syntax = match media_type { + MediaType::JavaScript => Syntax::Es(get_default_es_config()), + MediaType::JSX => { + let mut config = get_default_es_config(); + config.jsx = true; + Syntax::Es(config) + } + MediaType::TypeScript => Syntax::Typescript(get_default_ts_config()), + MediaType::TSX => { + let mut config = get_default_ts_config(); + config.tsx = true; + Syntax::Typescript(config) + } + _ => Syntax::Es(get_default_es_config()), + }; let lexer = Lexer::new( session, @@ -416,6 +451,7 @@ fn get_deno_types(parser: &AstParser, span: Span) -> Option { pub struct ImportDescriptor { pub specifier: String, pub deno_types: Option, + pub location: Location, } #[derive(Clone, Debug, PartialEq)] @@ -429,10 +465,12 @@ pub enum TsReferenceKind { pub struct TsReferenceDescriptor { pub kind: TsReferenceKind, pub specifier: String, + pub location: Location, } pub fn analyze_dependencies_and_references( file_name: &str, + media_type: MediaType, source_code: &str, analyze_dynamic_imports: bool, ) -> Result< @@ -440,7 +478,7 @@ pub fn analyze_dependencies_and_references( SwcDiagnosticBuffer, > { let parser = AstParser::new(); - parser.parse_module(file_name, source_code, |parse_result| { + parser.parse_module(file_name, media_type, source_code, |parse_result| { let module = parse_result?; let mut collector = NewDependencyVisitor { dependencies: vec![], @@ -461,16 +499,19 @@ pub fn analyze_dependencies_and_references( desc.kind != DependencyKind::DynamicImport }) .map(|desc| { + let location = parser.get_span_location(desc.span); if desc.kind == DependencyKind::Import { let deno_types = get_deno_types(&parser, desc.span); ImportDescriptor { specifier: desc.specifier.to_string(), deno_types, + location: location.into(), } } else { ImportDescriptor { specifier: desc.specifier.to_string(), deno_types: None, + location: location.into(), } } }) @@ -480,7 +521,7 @@ pub fn analyze_dependencies_and_references( let comments = parser .comments .take_leading_comments(module_span.lo()) - .unwrap_or_else(|| vec![]); + .unwrap_or_else(Vec::new); let mut references = vec![]; for comment in comments { @@ -518,7 +559,12 @@ pub fn analyze_dependencies_and_references( .trim_end_matches('\'') .to_string(); - references.push(TsReferenceDescriptor { kind, specifier }); + let location = parser.get_span_location(comment.span); + references.push(TsReferenceDescriptor { + kind, + specifier, + location: location.into(), + }); } Ok((imports, references)) }) @@ -547,24 +593,43 @@ console.log(fizz); console.log(qat.qat); "#; - let (imports, references) = - analyze_dependencies_and_references("some/file.ts", source, true) - .expect("Failed to parse"); + let (imports, references) = analyze_dependencies_and_references( + "some/file.ts", + MediaType::TypeScript, + source, + true, + ) + .expect("Failed to parse"); assert_eq!( imports, vec![ ImportDescriptor { specifier: "./type_definitions/foo.js".to_string(), - deno_types: Some("./type_definitions/foo.d.ts".to_string()) + deno_types: Some("./type_definitions/foo.d.ts".to_string()), + location: Location { + filename: "some/file.ts".to_string(), + line: 9, + col: 0, + }, }, ImportDescriptor { specifier: "./type_definitions/fizz.js".to_string(), - deno_types: Some("./type_definitions/fizz.d.ts".to_string()) + deno_types: Some("./type_definitions/fizz.d.ts".to_string()), + location: Location { + filename: "some/file.ts".to_string(), + line: 11, + col: 0, + }, }, ImportDescriptor { specifier: "./type_definitions/qat.ts".to_string(), - deno_types: None + deno_types: None, + location: Location { + filename: "some/file.ts".to_string(), + line: 15, + col: 0, + }, }, ] ); @@ -578,14 +643,29 @@ console.log(qat.qat); TsReferenceDescriptor { specifier: "dom".to_string(), kind: TsReferenceKind::Lib, + location: Location { + filename: "some/file.ts".to_string(), + line: 5, + col: 0, + }, }, TsReferenceDescriptor { specifier: "./type_reference.d.ts".to_string(), kind: TsReferenceKind::Types, + location: Location { + filename: "some/file.ts".to_string(), + line: 6, + col: 0, + }, }, TsReferenceDescriptor { specifier: "./type_reference/dep.ts".to_string(), kind: TsReferenceKind::Path, + location: Location { + filename: "some/file.ts".to_string(), + line: 7, + col: 0, + }, }, ] ); diff --git a/cli/test_runner.rs b/cli/test_runner.rs index 3e3ed1291ba456..b4623fc9f80db4 100644 --- a/cli/test_runner.rs +++ b/cli/test_runner.rs @@ -14,14 +14,17 @@ fn is_supported(p: &Path) -> bool { basename.ends_with("_test.ts") || basename.ends_with("_test.tsx") || basename.ends_with("_test.js") + || basename.ends_with("_test.mjs") || basename.ends_with("_test.jsx") || basename.ends_with(".test.ts") || basename.ends_with(".test.tsx") || basename.ends_with(".test.js") + || basename.ends_with(".test.mjs") || basename.ends_with(".test.jsx") || basename == "test.ts" || basename == "test.tsx" || basename == "test.js" + || basename == "test.mjs" || basename == "test.jsx" } else { false diff --git a/cli/tests/042_dyn_import_evalcontext.ts b/cli/tests/042_dyn_import_evalcontext.ts index 124a406d24de7b..e1cc485d147faa 100644 --- a/cli/tests/042_dyn_import_evalcontext.ts +++ b/cli/tests/042_dyn_import_evalcontext.ts @@ -1,4 +1,4 @@ -// @ts-ignore +// @ts-expect-error Deno.core.evalContext( "(async () => console.log(await import('./subdir/mod4.js')))()" ); diff --git a/cli/tests/056_make_temp_file_write_perm.ts b/cli/tests/056_make_temp_file_write_perm.ts index 15aefaff996cd0..c0deda8a25c305 100644 --- a/cli/tests/056_make_temp_file_write_perm.ts +++ b/cli/tests/056_make_temp_file_write_perm.ts @@ -1,8 +1,9 @@ -const path = await Deno.makeTempFile({ dir: "./subdir/" }); -if (path.startsWith(Deno.cwd())) { +const path = await Deno.makeTempFile({ dir: `subdir` }); +try { + if (!path.match(/^subdir[/\\][^/\\]+/)) { + throw Error("bad " + path); + } console.log("good", path); -} else { - throw Error("bad " + path); +} finally { + await Deno.remove(path); } -console.log(path); -await Deno.remove(path); diff --git a/cli/tests/059_fs_relative_path_perm.ts b/cli/tests/059_fs_relative_path_perm.ts new file mode 100644 index 00000000000000..26630fe1c4652d --- /dev/null +++ b/cli/tests/059_fs_relative_path_perm.ts @@ -0,0 +1,2 @@ +// The permission error message shouldn't include the CWD. +Deno.readFileSync("non-existent"); diff --git a/cli/tests/059_fs_relative_path_perm.ts.out b/cli/tests/059_fs_relative_path_perm.ts.out new file mode 100644 index 00000000000000..83df4190397de0 --- /dev/null +++ b/cli/tests/059_fs_relative_path_perm.ts.out @@ -0,0 +1,2 @@ +[WILDCARD]error: Uncaught PermissionDenied: read access to "non-existent", run again with the --allow-read flag + at [WILDCARD] diff --git a/cli/tests/bundle.test.out b/cli/tests/bundle.test.out index c3e12e8a914c3e..b286dad485b0cc 100644 --- a/cli/tests/bundle.test.out +++ b/cli/tests/bundle.test.out @@ -6,17 +6,13 @@ let System, __instantiateAsync, __instantiate; })(); System.register("print_hello", [], function (exports_1, context_1) { - [WILDCARD] +[WILDCARD] +}); +System.register("subdir2/mod2", ["print_hello"], function (exports_2, context_2) { +[WILDCARD] }); -System.register( - "subdir2/mod2", - ["print_hello"], - function (exports_2, context_2) { - [WILDCARD] - }, -); System.register("mod1", ["subdir2/mod2"], function (exports_3, context_3) { - [WILDCARD] +[WILDCARD] }); const __exp = __instantiate("mod1"); diff --git a/cli/tests/cjs_imports.ts b/cli/tests/cjs_imports.ts new file mode 100644 index 00000000000000..d8b77c22ebff69 --- /dev/null +++ b/cli/tests/cjs_imports.ts @@ -0,0 +1 @@ +import "./commonjs.cjs"; diff --git a/cli/tests/cjs_imports.ts.out b/cli/tests/cjs_imports.ts.out new file mode 100644 index 00000000000000..557db03de997c8 --- /dev/null +++ b/cli/tests/cjs_imports.ts.out @@ -0,0 +1 @@ +Hello World diff --git a/cli/tests/commonjs.cjs b/cli/tests/commonjs.cjs new file mode 100644 index 00000000000000..7df7d571e68fe2 --- /dev/null +++ b/cli/tests/commonjs.cjs @@ -0,0 +1 @@ +console.log("Hello World"); \ No newline at end of file diff --git a/cli/tests/disallow_http_from_https_js.out b/cli/tests/disallow_http_from_https_js.out index 7b71cb6bfebc3d..e4e42115905852 100644 --- a/cli/tests/disallow_http_from_https_js.out +++ b/cli/tests/disallow_http_from_https_js.out @@ -1 +1,2 @@ error: Modules loaded over https:// are not allowed to import modules over http:// +Imported from "https://localhost:5545/cli/tests/disallow_http_from_https.js:2" diff --git a/cli/tests/disallow_http_from_https_ts.out b/cli/tests/disallow_http_from_https_ts.out index 7b71cb6bfebc3d..55e10b73336521 100644 --- a/cli/tests/disallow_http_from_https_ts.out +++ b/cli/tests/disallow_http_from_https_ts.out @@ -1 +1,2 @@ error: Modules loaded over https:// are not allowed to import modules over http:// +Imported from "https://localhost:5545/cli/tests/disallow_http_from_https.ts:2" diff --git a/cli/tests/error_005_missing_dynamic_import.ts.out b/cli/tests/error_005_missing_dynamic_import.ts.out index 346e8cd6f99483..8a64175ec4dd1e 100644 --- a/cli/tests/error_005_missing_dynamic_import.ts.out +++ b/cli/tests/error_005_missing_dynamic_import.ts.out @@ -1 +1 @@ -error: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts" +error: Uncaught TypeError: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts" diff --git a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out index 7bebeda12f2519..78770dc817aa02 100644 --- a/cli/tests/error_012_bad_dynamic_import_specifier.ts.out +++ b/cli/tests/error_012_bad_dynamic_import_specifier.ts.out @@ -1 +1,2 @@ -error: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts" +Compile [WILDCARD]error_012_bad_dynamic_import_specifier.ts +error: Uncaught TypeError: relative import path "bad-module.ts" not prefixed with / or ./ or ../ Imported from "[WILDCARD]/error_012_bad_dynamic_import_specifier.ts" diff --git a/cli/tests/error_local_static_import_from_remote.js.out b/cli/tests/error_local_static_import_from_remote.js.out index c3fda12744bf95..1a2dcb2e3ea943 100644 --- a/cli/tests/error_local_static_import_from_remote.js.out +++ b/cli/tests/error_local_static_import_from_remote.js.out @@ -1,2 +1,3 @@ [WILDCARD] error: Remote modules are not allowed to statically import local modules. Use dynamic import instead. +Imported from "[WILDCARD]error_local_static_import_from_remote.js:1" diff --git a/cli/tests/error_local_static_import_from_remote.ts.out b/cli/tests/error_local_static_import_from_remote.ts.out index c3fda12744bf95..a2f2e1bbf1e79d 100644 --- a/cli/tests/error_local_static_import_from_remote.ts.out +++ b/cli/tests/error_local_static_import_from_remote.ts.out @@ -1,2 +1,3 @@ [WILDCARD] error: Remote modules are not allowed to statically import local modules. Use dynamic import instead. +Imported from "[WILDCARD]error_local_static_import_from_remote.ts:1" diff --git a/cli/tests/error_syntax.js.out b/cli/tests/error_syntax.js.out index 202e04a32cd4a4..107bc1df0ddd1a 100644 --- a/cli/tests/error_syntax.js.out +++ b/cli/tests/error_syntax.js.out @@ -1,4 +1 @@ -error: Uncaught SyntaxError: Unexpected identifier -(the following is a syntax error ^^ ! ) - ~~~~~~~~~ - at [WILDCARD]tests/error_syntax.js:3:6 +error: Expected Comma, got Some(Word(following)) at [WILDCARD]tests/error_syntax.js:3:5 diff --git a/cli/tests/error_syntax_empty_trailing_line.mjs.out b/cli/tests/error_syntax_empty_trailing_line.mjs.out index 6e8a268e9d8889..3b78a23a65f60a 100644 --- a/cli/tests/error_syntax_empty_trailing_line.mjs.out +++ b/cli/tests/error_syntax_empty_trailing_line.mjs.out @@ -1,2 +1 @@ -error: Uncaught SyntaxError: Unexpected end of input - at [WILDCARD]tests/error_syntax_empty_trailing_line.mjs:[WILDCARD] +error: Unexpected eof at [WILDCARD]tests/error_syntax_empty_trailing_line.mjs:2:21 diff --git a/cli/tests/es_private_fields.js b/cli/tests/es_private_fields.js new file mode 100644 index 00000000000000..b5f83e39c79913 --- /dev/null +++ b/cli/tests/es_private_fields.js @@ -0,0 +1,15 @@ +class Foo { + #field = "field"; + + setValue(val) { + this.#field = val; + } + + getValue() { + return this.#field; + } +} + +const bar = new Foo(); +bar.setValue("PRIVATE"); +console.log(bar.getValue()); diff --git a/cli/tests/es_private_fields.js.out b/cli/tests/es_private_fields.js.out new file mode 100644 index 00000000000000..be1970b05340d6 --- /dev/null +++ b/cli/tests/es_private_fields.js.out @@ -0,0 +1 @@ +PRIVATE diff --git a/cli/tests/file_exists.ts b/cli/tests/file_exists.ts new file mode 100644 index 00000000000000..5fc5414b389967 --- /dev/null +++ b/cli/tests/file_exists.ts @@ -0,0 +1,6 @@ +try { + await Deno.open(Deno.args[0]); + Deno.exit(0); +} catch (e) { + Deno.exit(1); +} diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index b3f792b4e9bf3b..450c2653876164 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -53,6 +53,23 @@ fn x_deno_warning() { drop(g); } +#[test] +fn eval_p() { + let output = util::deno_cmd() + .arg("eval") + .arg("-p") + .arg("1+2") + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(output.status.success()); + let stdout_str = + util::strip_ansi_codes(std::str::from_utf8(&output.stdout).unwrap().trim()); + assert_eq!("3", stdout_str); +} + #[test] fn no_color() { let output = util::deno_cmd() @@ -412,6 +429,71 @@ fn js_unit_tests() { assert!(status.success()); } +#[test] +fn ts_dependency_recompilation() { + let t = TempDir::new().expect("tempdir fail"); + let ats = t.path().join("a.ts"); + + std::fs::write( + &ats, + " + import { foo } from \"./b.ts\"; + + function print(str: string): void { + console.log(str); + } + + print(foo);", + ) + .unwrap(); + + let bts = t.path().join("b.ts"); + std::fs::write( + &bts, + " + export const foo = \"foo\";", + ) + .unwrap(); + + let output = util::deno_cmd() + .current_dir(util::root_path()) + .env("NO_COLOR", "1") + .arg("run") + .arg(&ats) + .output() + .expect("failed to spawn script"); + + let stdout_output = std::str::from_utf8(&output.stdout).unwrap().trim(); + let stderr_output = std::str::from_utf8(&output.stderr).unwrap().trim(); + + assert!(stdout_output.ends_with("foo")); + assert!(stderr_output.starts_with("Compile")); + + // Overwrite contents of b.ts and run again + std::fs::write( + &bts, + " + export const foo = 5;", + ) + .expect("error writing file"); + + let output = util::deno_cmd() + .current_dir(util::root_path()) + .env("NO_COLOR", "1") + .arg("run") + .arg(&ats) + .output() + .expect("failed to spawn script"); + + let stdout_output = std::str::from_utf8(&output.stdout).unwrap().trim(); + let stderr_output = std::str::from_utf8(&output.stderr).unwrap().trim(); + + // error: TS2345 [ERROR]: Argument of type '5' is not assignable to parameter of type 'string'. + assert!(stderr_output.contains("TS2345")); + assert!(!output.status.success()); + assert!(stdout_output.is_empty()); +} + #[test] fn bundle_exports() { // First we have to generate a bundle of some module that has exports. @@ -680,6 +762,18 @@ fn repl_test_console_log() { assert!(err.is_empty()); } +#[test] +fn repl_cwd() { + let (_out, err) = util::run_and_collect_output( + true, + "repl", + Some(vec!["Deno.cwd()"]), + Some(vec![("NO_COLOR".to_owned(), "1".to_owned())]), + false, + ); + assert!(err.is_empty()); +} + #[test] fn repl_test_eof() { let (out, err) = util::run_and_collect_output( @@ -693,6 +787,24 @@ fn repl_test_eof() { assert!(err.is_empty()); } +#[test] +fn repl_test_strict() { + let (_, err) = util::run_and_collect_output( + true, + "repl", + Some(vec![ + "let a = {};", + "Object.preventExtensions(a);", + "a.c = 1;", + ]), + None, + false, + ); + assert!(err.contains( + "Uncaught TypeError: Cannot add property c, object is not extensible" + )); +} + const REPL_MSG: &str = "exit using ctrl+d or close()\n"; #[test] @@ -1243,6 +1355,12 @@ itest!(_058_tasks_microtasks_close { output: "058_tasks_microtasks_close.ts.out", }); +itest!(_059_fs_relative_path_perm { + args: "run 059_fs_relative_path_perm.ts", + output: "059_fs_relative_path_perm.ts.out", + exit_code: 1, +}); + itest!(js_import_detect { args: "run --quiet --reload js_import_detect.ts", output: "js_import_detect.ts.out", @@ -1359,7 +1477,7 @@ itest!(error_004_missing_module { }); itest!(error_005_missing_dynamic_import { - args: "run --reload --allow-read error_005_missing_dynamic_import.ts", + args: "run --reload --allow-read --quiet error_005_missing_dynamic_import.ts", exit_code: 1, output: "error_005_missing_dynamic_import.ts.out", }); @@ -1406,7 +1524,7 @@ itest!(error_014_catch_dynamic_import_error { }); itest!(error_015_dynamic_import_permissions { - args: "run --reload error_015_dynamic_import_permissions.js", + args: "run --reload --quiet error_015_dynamic_import_permissions.js", output: "error_015_dynamic_import_permissions.out", exit_code: 1, http_server: true, @@ -1738,6 +1856,21 @@ itest!(fix_js_imports { output: "fix_js_imports.ts.out", }); +itest!(es_private_fields { + args: "run --quiet --reload es_private_fields.js", + output: "es_private_fields.js.out", +}); + +itest!(cjs_imports { + args: "run --quiet --reload cjs_imports.ts", + output: "cjs_imports.ts.out", +}); + +itest!(ts_import_from_js { + args: "run --quiet --reload ts_import_from_js.js", + output: "ts_import_from_js.js.out", +}); + itest!(proto_exploit { args: "run proto_exploit.js", output: "proto_exploit.js.out", @@ -1862,14 +1995,17 @@ fn cafile_bundle_remote_exports() { #[test] fn test_permissions_with_allow() { for permission in &util::PERMISSION_VARIANTS { - let (_, err) = util::run_and_collect_output( - true, - &format!("run --allow-{0} permission_test.ts {0}Required", permission), - None, - None, - false, - ); - assert!(!err.contains(util::PERMISSION_DENIED_PATTERN)); + let status = util::deno_cmd() + .current_dir(&util::tests_path()) + .arg("run") + .arg(format!("--allow-{0}", permission)) + .arg("permission_test.ts") + .arg(format!("{0}Required", permission)) + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); } } @@ -1891,19 +2027,22 @@ fn test_permissions_without_allow() { fn test_permissions_rw_inside_project_dir() { const PERMISSION_VARIANTS: [&str; 2] = ["read", "write"]; for permission in &PERMISSION_VARIANTS { - let (_, err) = util::run_and_collect_output( - true, - &format!( - "run --allow-{0}={1} complex_permissions_test.ts {0} {2} {2}", + let status = util::deno_cmd() + .current_dir(&util::tests_path()) + .arg("run") + .arg(format!( + "--allow-{0}={1}", permission, - util::root_path().into_os_string().into_string().unwrap(), - "complex_permissions_test.ts" - ), - None, - None, - false, - ); - assert!(!err.contains(util::PERMISSION_DENIED_PATTERN)); + util::root_path().into_os_string().into_string().unwrap() + )) + .arg("complex_permissions_test.ts") + .arg(permission) + .arg("complex_permissions_test.ts") + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); } } @@ -1940,24 +2079,27 @@ fn test_permissions_rw_outside_test_dir() { fn test_permissions_rw_inside_test_dir() { const PERMISSION_VARIANTS: [&str; 2] = ["read", "write"]; for permission in &PERMISSION_VARIANTS { - let (_, err) = util::run_and_collect_output( - true, - &format!( - "run --allow-{0}={1} complex_permissions_test.ts {0} {2}", + let status = util::deno_cmd() + .current_dir(&util::tests_path()) + .arg("run") + .arg(format!( + "--allow-{0}={1}", permission, util::root_path() .join("cli") .join("tests") .into_os_string() .into_string() - .unwrap(), - "complex_permissions_test.ts" - ), - None, - None, - false, - ); - assert!(!err.contains(util::PERMISSION_DENIED_PATTERN)); + .unwrap() + )) + .arg("complex_permissions_test.ts") + .arg(permission) + .arg("complex_permissions_test.ts") + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); } } @@ -2012,17 +2154,18 @@ fn test_permissions_rw_inside_test_and_js_dir() { .into_string() .unwrap(); for permission in &PERMISSION_VARIANTS { - let (_, err) = util::run_and_collect_output( - true, - &format!( - "run --allow-{0}={1},{2} complex_permissions_test.ts {0} {3}", - permission, test_dir, js_dir, "complex_permissions_test.ts" - ), - None, - None, - false, - ); - assert!(!err.contains(util::PERMISSION_DENIED_PATTERN)); + let status = util::deno_cmd() + .current_dir(&util::tests_path()) + .arg("run") + .arg(format!("--allow-{0}={1},{2}", permission, test_dir, js_dir)) + .arg("complex_permissions_test.ts") + .arg(permission) + .arg("complex_permissions_test.ts") + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); } } @@ -2030,17 +2173,18 @@ fn test_permissions_rw_inside_test_and_js_dir() { fn test_permissions_rw_relative() { const PERMISSION_VARIANTS: [&str; 2] = ["read", "write"]; for permission in &PERMISSION_VARIANTS { - let (_, err) = util::run_and_collect_output( - true, - &format!( - "run --allow-{0}=. complex_permissions_test.ts {0} complex_permissions_test.ts", - permission - ), - None, - None, - false, - ); - assert!(!err.contains(util::PERMISSION_DENIED_PATTERN)); + let status = util::deno_cmd() + .current_dir(&util::tests_path()) + .arg("run") + .arg(format!("--allow-{0}=.", permission)) + .arg("complex_permissions_test.ts") + .arg(permission) + .arg("complex_permissions_test.ts") + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); } } @@ -2048,17 +2192,18 @@ fn test_permissions_rw_relative() { fn test_permissions_rw_no_prefix() { const PERMISSION_VARIANTS: [&str; 2] = ["read", "write"]; for permission in &PERMISSION_VARIANTS { - let (_, err) = util::run_and_collect_output( - true, - &format!( - "run --allow-{0}=tls/../ complex_permissions_test.ts {0} complex_permissions_test.ts", - permission - ), - None, - None, - false, - ); - assert!(!err.contains(util::PERMISSION_DENIED_PATTERN)); + let status = util::deno_cmd() + .current_dir(&util::tests_path()) + .arg("run") + .arg(format!("--allow-{0}=tls/../", permission)) + .arg("complex_permissions_test.ts") + .arg(permission) + .arg("complex_permissions_test.ts") + .spawn() + .unwrap() + .wait() + .unwrap(); + assert!(status.success()); } } @@ -2207,12 +2352,17 @@ fn test_permissions_net_listen_allow_localhost() { assert!(!err.contains(util::PERMISSION_DENIED_PATTERN)); } +fn inspect_flag_with_unique_port(flag_prefix: &str) -> String { + use std::sync::atomic::{AtomicU16, Ordering}; + static PORT: AtomicU16 = AtomicU16::new(9229); + let port = PORT.fetch_add(1, Ordering::Relaxed); + format!("{}=127.0.0.1:{}", flag_prefix, port) +} + fn extract_ws_url_from_stderr( - stderr: &mut std::process::ChildStderr, + stderr_lines: &mut impl std::iter::Iterator, ) -> url::Url { - let mut stderr = std::io::BufReader::new(stderr); - let mut stderr_first_line = String::from(""); - let _ = stderr.read_line(&mut stderr_first_line).unwrap(); + let stderr_first_line = stderr_lines.next().unwrap(); assert!(stderr_first_line.starts_with("Debugger listening on ")); let v: Vec<_> = stderr_first_line.match_indices("ws:").collect(); assert_eq!(v.len(), 1); @@ -2226,15 +2376,17 @@ async fn inspector_connect() { let script = util::tests_path().join("inspector1.js"); let mut child = util::deno_cmd() .arg("run") - // Warning: each inspector test should be on its own port to avoid - // conflicting with another inspector test. - .arg("--inspect=127.0.0.1:9228") + .arg(inspect_flag_with_unique_port("--inspect")) .arg(script) .stderr(std::process::Stdio::piped()) .spawn() .unwrap(); - let ws_url = extract_ws_url_from_stderr(child.stderr.as_mut().unwrap()); - println!("ws_url {}", ws_url); + + let stderr = child.stderr.as_mut().unwrap(); + let mut stderr_lines = + std::io::BufReader::new(stderr).lines().map(|r| r.unwrap()); + let ws_url = extract_ws_url_from_stderr(&mut stderr_lines); + // We use tokio_tungstenite as a websocket client because warp (which is // a dependency of Deno) uses it. let (_socket, response) = tokio_tungstenite::connect_async(ws_url) @@ -2247,6 +2399,7 @@ async fn inspector_connect() { enum TestStep { StdOut(&'static str), + StdErr(&'static str), WsRecv(&'static str), WsSend(&'static str), } @@ -2256,9 +2409,7 @@ async fn inspector_break_on_first_line() { let script = util::tests_path().join("inspector2.js"); let mut child = util::deno_cmd() .arg("run") - // Warning: each inspector test should be on its own port to avoid - // conflicting with another inspector test. - .arg("--inspect-brk=127.0.0.1:9229") + .arg(inspect_flag_with_unique_port("--inspect-brk")) .arg(script) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) @@ -2266,7 +2417,10 @@ async fn inspector_break_on_first_line() { .unwrap(); let stderr = child.stderr.as_mut().unwrap(); - let ws_url = extract_ws_url_from_stderr(stderr); + let mut stderr_lines = + std::io::BufReader::new(stderr).lines().map(|r| r.unwrap()); + let ws_url = extract_ws_url_from_stderr(&mut stderr_lines); + let (socket, response) = tokio_tungstenite::connect_async(ws_url) .await .expect("Can't connect"); @@ -2310,6 +2464,7 @@ async fn inspector_break_on_first_line() { StdOut(s) => assert_eq!(&stdout_lines.next().unwrap(), s), WsRecv(s) => assert!(socket_rx.next().await.unwrap().starts_with(s)), WsSend(s) => socket_tx.send(s.into()).await.unwrap(), + _ => unreachable!(), } } @@ -2322,14 +2477,17 @@ async fn inspector_pause() { let script = util::tests_path().join("inspector1.js"); let mut child = util::deno_cmd() .arg("run") - // Warning: each inspector test should be on its own port to avoid - // conflicting with another inspector test. - .arg("--inspect=127.0.0.1:9230") + .arg(inspect_flag_with_unique_port("--inspect")) .arg(script) .stderr(std::process::Stdio::piped()) .spawn() .unwrap(); - let ws_url = extract_ws_url_from_stderr(child.stderr.as_mut().unwrap()); + + let stderr = child.stderr.as_mut().unwrap(); + let mut stderr_lines = + std::io::BufReader::new(stderr).lines().map(|r| r.unwrap()); + let ws_url = extract_ws_url_from_stderr(&mut stderr_lines); + // We use tokio_tungstenite as a websocket client because warp (which is // a dependency of Deno) uses it. let (mut socket, _) = tokio_tungstenite::connect_async(ws_url) @@ -2376,19 +2534,25 @@ async fn inspector_pause() { #[tokio::test] async fn inspector_port_collision() { let script = util::tests_path().join("inspector1.js"); + let inspect_flag = inspect_flag_with_unique_port("--inspect"); + let mut child1 = util::deno_cmd() .arg("run") - .arg("--inspect=127.0.0.1:9231") + .arg(&inspect_flag) .arg(script.clone()) .stderr(std::process::Stdio::piped()) .spawn() .unwrap(); - let ws_url_1 = extract_ws_url_from_stderr(child1.stderr.as_mut().unwrap()); - println!("ws_url {}", ws_url_1); + + let stderr_1 = child1.stderr.as_mut().unwrap(); + let mut stderr_lines_1 = std::io::BufReader::new(stderr_1) + .lines() + .map(|r| r.unwrap()); + let _ = extract_ws_url_from_stderr(&mut stderr_lines_1); let mut child2 = util::deno_cmd() .arg("run") - .arg("--inspect=127.0.0.1:9231") + .arg(&inspect_flag) .arg(script) .stderr(std::process::Stdio::piped()) .spawn() @@ -2414,9 +2578,7 @@ async fn inspector_does_not_hang() { let script = util::tests_path().join("inspector3.js"); let mut child = util::deno_cmd() .arg("run") - // Warning: each inspector test should be on its own port to avoid - // conflicting with another inspector test. - .arg("--inspect-brk=127.0.0.1:9232") + .arg(inspect_flag_with_unique_port("--inspect-brk")) .env("NO_COLOR", "1") .arg(script) .stdout(std::process::Stdio::piped()) @@ -2425,7 +2587,10 @@ async fn inspector_does_not_hang() { .unwrap(); let stderr = child.stderr.as_mut().unwrap(); - let ws_url = extract_ws_url_from_stderr(stderr); + let mut stderr_lines = + std::io::BufReader::new(stderr).lines().map(|r| r.unwrap()); + let ws_url = extract_ws_url_from_stderr(&mut stderr_lines); + let (socket, response) = tokio_tungstenite::connect_async(ws_url) .await .expect("Can't connect"); @@ -2498,24 +2663,106 @@ async fn inspector_without_brk_runs_code() { let script = util::tests_path().join("inspector4.js"); let mut child = util::deno_cmd() .arg("run") - // Warning: each inspector test should be on its own port to avoid - // conflicting with another inspector test. - .arg("--inspect=127.0.0.1:9233") + .arg(inspect_flag_with_unique_port("--inspect")) .arg(script) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn() .unwrap(); - extract_ws_url_from_stderr(child.stderr.as_mut().unwrap()); + + let stderr = child.stderr.as_mut().unwrap(); + let mut stderr_lines = + std::io::BufReader::new(stderr).lines().map(|r| r.unwrap()); + let _ = extract_ws_url_from_stderr(&mut stderr_lines); // Check that inspector actually runs code without waiting for inspector - // connection - let mut stdout = std::io::BufReader::new(child.stdout.as_mut().unwrap()); - let mut stdout_first_line = String::from(""); - let _ = stdout.read_line(&mut stdout_first_line).unwrap(); - assert_eq!(stdout_first_line, "hello\n"); + // connection. + let stdout = child.stdout.as_mut().unwrap(); + let mut stdout_lines = + std::io::BufReader::new(stdout).lines().map(|r| r.unwrap()); + let stdout_first_line = stdout_lines.next().unwrap(); + assert_eq!(stdout_first_line, "hello"); child.kill().unwrap(); + child.wait().unwrap(); +} + +#[tokio::test] +async fn inspector_runtime_evaluate_does_not_crash() { + let mut child = util::deno_cmd() + .arg("repl") + .arg(inspect_flag_with_unique_port("--inspect")) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let stderr = child.stderr.as_mut().unwrap(); + let mut stderr_lines = std::io::BufReader::new(stderr) + .lines() + .map(|r| r.unwrap()) + .filter(|s| s.as_str() != "Debugger session started."); + let ws_url = extract_ws_url_from_stderr(&mut stderr_lines); + + let (socket, response) = tokio_tungstenite::connect_async(ws_url) + .await + .expect("Can't connect"); + assert_eq!(response.status(), 101); // Switching protocols. + + let (mut socket_tx, socket_rx) = socket.split(); + let mut socket_rx = + socket_rx.map(|msg| msg.unwrap().to_string()).filter(|msg| { + let pass = !msg.starts_with(r#"{"method":"Debugger.scriptParsed","#); + futures::future::ready(pass) + }); + + let stdin = child.stdin.take().unwrap(); + + let stdout = child.stdout.as_mut().unwrap(); + let mut stdout_lines = std::io::BufReader::new(stdout) + .lines() + .map(|r| r.unwrap()) + .filter(|s| !s.starts_with("Deno ")); + + use TestStep::*; + let test_steps = vec![ + WsSend(r#"{"id":1,"method":"Runtime.enable"}"#), + WsSend(r#"{"id":2,"method":"Debugger.enable"}"#), + WsRecv( + r#"{"method":"Runtime.executionContextCreated","params":{"context":{"id":1,"#, + ), + WsRecv(r#"{"id":1,"result":{}}"#), + WsRecv(r#"{"id":2,"result":{"debuggerId":"#), + WsSend(r#"{"id":3,"method":"Runtime.runIfWaitingForDebugger"}"#), + WsRecv(r#"{"id":3,"result":{}}"#), + StdOut("exit using ctrl+d or close()"), + WsSend( + r#"{"id":4,"method":"Runtime.compileScript","params":{"expression":"Deno.cwd()","sourceURL":"","persistScript":false,"executionContextId":1}}"#, + ), + WsRecv(r#"{"id":4,"result":{}}"#), + WsSend( + r#"{"id":5,"method":"Runtime.evaluate","params":{"expression":"Deno.cwd()","objectGroup":"console","includeCommandLineAPI":true,"silent":false,"contextId":1,"returnByValue":true,"generatePreview":true,"userGesture":true,"awaitPromise":false,"replMode":true}}"#, + ), + WsRecv(r#"{"id":5,"result":{"result":{"type":"string","value":""#), + WsSend( + r#"{"id":6,"method":"Runtime.evaluate","params":{"expression":"console.error('done');","objectGroup":"console","includeCommandLineAPI":true,"silent":false,"contextId":1,"returnByValue":true,"generatePreview":true,"userGesture":true,"awaitPromise":false,"replMode":true}}"#, + ), + WsRecv(r#"{"id":6,"result":{"result":{"type":"undefined"}}}"#), + StdErr("done"), + ]; + + for step in test_steps { + match step { + StdOut(s) => assert_eq!(&stdout_lines.next().unwrap(), s), + StdErr(s) => assert_eq!(&stderr_lines.next().unwrap(), s), + WsRecv(s) => assert!(socket_rx.next().await.unwrap().starts_with(s)), + WsSend(s) => socket_tx.send(s.into()).await.unwrap(), + } + } + + std::mem::drop(stdin); + child.wait().unwrap(); } #[test] @@ -2557,7 +2804,7 @@ mod util { pub const PERMISSION_DENIED_PATTERN: &str = "PermissionDenied"; lazy_static! { - static ref DENO_DIR: TempDir = { TempDir::new().expect("tempdir fail") }; + static ref DENO_DIR: TempDir = TempDir::new().expect("tempdir fail"); // STRIP_ANSI_RE and strip_ansi_codes are lifted from the "console" crate. // Copyright 2017 Armin Ronacher . MIT License. diff --git a/cli/tests/lock_write_fetch.ts b/cli/tests/lock_write_fetch.ts index 9e719059652c95..5a4dea0ce94cce 100644 --- a/cli/tests/lock_write_fetch.ts +++ b/cli/tests/lock_write_fetch.ts @@ -32,6 +32,8 @@ const fetchCheckProc = Deno.run({ const fetchCheckProcCode = (await fetchCheckProc.status()).code; console.log(`fetch check code: ${fetchCheckProcCode}`); +Deno.removeSync("./lock_write_fetch.json"); + const runProc = Deno.run({ stdout: "null", stderr: "null", @@ -39,7 +41,10 @@ const runProc = Deno.run({ Deno.execPath(), "run", "--lock=lock_write_fetch.json", - "https_import.ts", + "--lock-write", + "--allow-read", + "file_exists.ts", + "lock_write_fetch.json", ], }); diff --git a/cli/tests/ts_import_from_js.js b/cli/tests/ts_import_from_js.js new file mode 100644 index 00000000000000..e06ca15a2480a4 --- /dev/null +++ b/cli/tests/ts_import_from_js.js @@ -0,0 +1 @@ +import "./005_more_imports.ts"; diff --git a/cli/tests/ts_import_from_js.js.out b/cli/tests/ts_import_from_js.js.out new file mode 100644 index 00000000000000..e965047ad7c578 --- /dev/null +++ b/cli/tests/ts_import_from_js.js.out @@ -0,0 +1 @@ +Hello diff --git a/cli/tests/ts_with_generic.ts b/cli/tests/ts_with_generic.ts new file mode 100644 index 00000000000000..aa83e73b9fb691 --- /dev/null +++ b/cli/tests/ts_with_generic.ts @@ -0,0 +1,3 @@ +/* eslint-disable */ + +const foo = { delete() {} }; diff --git a/cli/tests/unit/blob_test.ts b/cli/tests/unit/blob_test.ts index 70f8fb3d5a2dc9..79e6f1f8f00ccd 100644 --- a/cli/tests/unit/blob_test.ts +++ b/cli/tests/unit/blob_test.ts @@ -90,3 +90,8 @@ unitTest(async function blobStream(): Promise { await read(); assertEquals(decode(bytes), "Hello World"); }); + +unitTest(function blobConstructorNameIsBlob(): void { + const blob = new Blob(); + assertEquals(blob.constructor.name, "Blob"); +}); diff --git a/cli/tests/unit/body_test.ts b/cli/tests/unit/body_test.ts index c8f783e0430eab..a6df3102e67c68 100644 --- a/cli/tests/unit/body_test.ts +++ b/cli/tests/unit/body_test.ts @@ -42,8 +42,8 @@ unitTest( const body = buildBody(text); - // @ts-ignore - body.contentType = "multipart/form-data;boundary=boundary"; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (body as any).contentType = "multipart/form-data;boundary=boundary"; const formData = await body.formData(); assert(formData.has("field_1")); @@ -62,8 +62,8 @@ unitTest( const body = buildBody(text); - // @ts-ignore - body.contentType = "application/x-www-form-urlencoded"; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (body as any).contentType = "application/x-www-form-urlencoded"; const formData = await body.formData(); assert(formData.has("field_1")); diff --git a/cli/tests/unit/buffer_test.ts b/cli/tests/unit/buffer_test.ts index ac2d7db7b9856a..a5a0147abd813d 100644 --- a/cli/tests/unit/buffer_test.ts +++ b/cli/tests/unit/buffer_test.ts @@ -6,7 +6,7 @@ import { assertEquals, assert, - assertStrContains, + assertStringContains, unitTest, } from "./test_util.ts"; @@ -164,7 +164,7 @@ unitTest(async function bufferTooLargeByteWrites(): Promise { } assert(err instanceof Error); - assertStrContains(err.message, "grown beyond the maximum size"); + assertStringContains(err.message, "grown beyond the maximum size"); }); unitTest(async function bufferLargeByteReads(): Promise { diff --git a/cli/tests/unit/console_test.ts b/cli/tests/unit/console_test.ts index 02d71e4ca12443..98e43c7395887e 100644 --- a/cli/tests/unit/console_test.ts +++ b/cli/tests/unit/console_test.ts @@ -22,7 +22,7 @@ const customInspect = Deno.customInspect; const { Console, stringifyArgs, - // @ts-ignore TypeScript (as of 3.7) does not support indexing namespaces by symbol + // @ts-expect-error TypeScript (as of 3.7) does not support indexing namespaces by symbol } = Deno[Deno.internal]; function stringify(...args: unknown[]): string { @@ -571,7 +571,7 @@ unitTest(function consoleTestStringifyIterable() { `[ <4 empty items>, 0, 0, <4 empty items> ]` ); - /* TODO(ry) Fix this test + /* TODO(ry) Fix this test const lWithEmptyEl = Array(200); lWithEmptyEl.fill(0, 50, 80); assertEquals( @@ -590,7 +590,7 @@ unitTest(function consoleTestStringifyIterable() { 0, <120 empty items> ]` ); -*/ + */ }); unitTest(async function consoleTestStringifyPromises(): Promise { @@ -727,7 +727,6 @@ unitTest(function consoleTestCallToStringOnLabel(): void { mockConsole((console) => { for (const method of methods) { let hasCalled = false; - // @ts-ignore console[method]({ toString(): void { hasCalled = true; @@ -1071,6 +1070,36 @@ unitTest(function consoleTable(): void { │ 1 │ "你好" │ │ 2 │ "Amapá" │ └───────┴─────────┘ +` + ); + }); + mockConsole((console, out): void => { + console.table([ + [1, 2], + [3, 4], + ]); + assertEquals( + stripColor(out.toString()), + `┌───────┬───┬───┐ +│ (idx) │ 0 │ 1 │ +├───────┼───┼───┤ +│ 0 │ 1 │ 2 │ +│ 1 │ 3 │ 4 │ +└───────┴───┴───┘ +` + ); + }); + mockConsole((console, out): void => { + console.table({ 1: { a: 4, b: 5 }, 2: null, 3: { b: 6, c: 7 } }, ["b"]); + assertEquals( + stripColor(out.toString()), + `┌───────┬───┐ +│ (idx) │ b │ +├───────┼───┤ +│ 1 │ 5 │ +│ 2 │ │ +│ 3 │ 6 │ +└───────┴───┘ ` ); }); diff --git a/cli/tests/unit/dir_test.ts b/cli/tests/unit/dir_test.ts index dc05d9564ba18a..5b60b3dbaa082c 100644 --- a/cli/tests/unit/dir_test.ts +++ b/cli/tests/unit/dir_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { unitTest, assert, assertEquals } from "./test_util.ts"; +import { unitTest, assert, assertEquals, assertThrows } from "./test_util.ts"; unitTest({ perms: { read: true } }, function dirCwdNotNull(): void { assert(Deno.cwd() != null); @@ -29,32 +29,31 @@ unitTest({ perms: { read: true, write: true } }, function dirCwdError(): void { Deno.chdir(path); Deno.removeSync(path); try { - Deno.cwd(); - throw Error("current directory removed, should throw error"); - } catch (err) { - if (err instanceof Deno.errors.NotFound) { - assert(err.name === "NotFound"); - } else { - throw Error("raised different exception"); - } + assertThrows(() => { + Deno.cwd(); + }, Deno.errors.NotFound); + } finally { + Deno.chdir(initialdir); } - Deno.chdir(initialdir); } }); +unitTest({ perms: { read: false } }, function dirCwdPermError(): void { + assertThrows( + () => { + Deno.cwd(); + }, + Deno.errors.PermissionDenied, + "read access to , run again with the --allow-read flag" + ); +}); + unitTest( { perms: { read: true, write: true } }, function dirChdirError(): void { const path = Deno.makeTempDirSync() + "test"; - try { + assertThrows(() => { Deno.chdir(path); - throw Error("directory not available, should throw error"); - } catch (err) { - if (err instanceof Deno.errors.NotFound) { - assert(err.name === "NotFound"); - } else { - throw Error("raised different exception"); - } - } + }, Deno.errors.NotFound); } ); diff --git a/cli/tests/unit/dispatch_json_test.ts b/cli/tests/unit/dispatch_json_test.ts index 4e95b86a2858f6..ebb8217e3694a2 100644 --- a/cli/tests/unit/dispatch_json_test.ts +++ b/cli/tests/unit/dispatch_json_test.ts @@ -19,14 +19,19 @@ unitTest( } ); +/* eslint-disable @typescript-eslint/no-namespace, @typescript-eslint/no-explicit-any,no-var */ +declare global { + namespace Deno { + var core: any; + } +} +/* eslint-enable */ + unitTest(function malformedJsonControlBuffer(): void { - // @ts-ignore const opId = Deno.core.ops()["op_open"]; - // @ts-ignore const res = Deno.core.send(opId, new Uint8Array([1, 2, 3, 4, 5])); const resText = new TextDecoder().decode(res); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const resJson = JSON.parse(resText) as any; + const resJson = JSON.parse(resText); assert(!resJson.ok); assert(resJson.err); }); diff --git a/cli/tests/unit/dispatch_minimal_test.ts b/cli/tests/unit/dispatch_minimal_test.ts index afc17f4fb8434b..12cf4595c7d7e9 100644 --- a/cli/tests/unit/dispatch_minimal_test.ts +++ b/cli/tests/unit/dispatch_minimal_test.ts @@ -25,10 +25,16 @@ unitTest(async function sendAsyncStackTrace(): Promise { } }); +/* eslint-disable @typescript-eslint/no-namespace, @typescript-eslint/no-explicit-any,no-var */ +declare global { + namespace Deno { + var core: any; + } +} +/* eslint-enable */ + unitTest(function malformedMinimalControlBuffer(): void { - // @ts-ignore const readOpId = Deno.core.ops()["op_read"]; - // @ts-ignore const res = Deno.core.send(readOpId, new Uint8Array([1, 2, 3, 4, 5])); const header = res.slice(0, 12); const buf32 = new Int32Array( diff --git a/cli/tests/unit/dom_iterable_test.ts b/cli/tests/unit/dom_iterable_test.ts index b9435b3bcc657e..c4e535a25759c5 100644 --- a/cli/tests/unit/dom_iterable_test.ts +++ b/cli/tests/unit/dom_iterable_test.ts @@ -20,7 +20,7 @@ function setup() { Base, // This is using an internal API we don't want published as types, so having // to cast to any to "trick" TypeScript - // @ts-ignore TypeScript (as of 3.7) does not support indexing namespaces by symbol + // @ts-expect-error TypeScript (as of 3.7) does not support indexing namespaces by symbol DomIterable: Deno[Deno.internal].DomIterableMixin(Base, dataSymbol), }; } diff --git a/cli/tests/unit/error_stack_test.ts b/cli/tests/unit/error_stack_test.ts index e5cedfcf51977a..052cb959103c0e 100644 --- a/cli/tests/unit/error_stack_test.ts +++ b/cli/tests/unit/error_stack_test.ts @@ -1,7 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { unitTest, assert } from "./test_util.ts"; +import { assert, assertEquals, unitTest } from "./test_util.ts"; -// @ts-ignore TypeScript (as of 3.7) does not support indexing namespaces by symbol +// @ts-expect-error TypeScript (as of 3.7) does not support indexing namespaces by symbol const { setPrepareStackTrace } = Deno[Deno.internal]; interface CallSite { @@ -96,6 +96,18 @@ unitTest(function prepareStackTrace(): void { assert(result.includes(".ts:"), "should remap to something in 'js/'"); }); +unitTest(function captureStackTrace(): void { + function foo(): void { + const error = new Error(); + const stack1 = error.stack!; + Error.captureStackTrace(error, foo); + const stack2 = error.stack!; + // stack2 should be stack1 without the first frame. + assertEquals(stack2, stack1.replace(/(?<=^[^\n]*\n)[^\n]*\n/, "")); + } + foo(); +}); + unitTest(function applySourceMap(): void { const result = Deno.applySourceMap({ fileName: "CLI_SNAPSHOT.js", diff --git a/cli/tests/unit/event_target_test.ts b/cli/tests/unit/event_target_test.ts index 0c4eb4d0ddc386..cfbe5285b2d824 100644 --- a/cli/tests/unit/event_target_test.ts +++ b/cli/tests/unit/event_target_test.ts @@ -4,11 +4,8 @@ import { unitTest, assertEquals } from "./test_util.ts"; unitTest(function addEventListenerTest(): void { const document = new EventTarget(); - // @ts-ignore tests ignoring the type system for resilience assertEquals(document.addEventListener("x", null, false), undefined); - // @ts-ignore assertEquals(document.addEventListener("x", null, true), undefined); - // @ts-ignore assertEquals(document.addEventListener("x", null), undefined); }); @@ -71,11 +68,8 @@ unitTest(function anEventTargetCanBeSubclassed(): void { unitTest(function removingNullEventListenerShouldSucceed(): void { const document = new EventTarget(); - // @ts-ignore assertEquals(document.removeEventListener("x", null, false), undefined); - // @ts-ignore assertEquals(document.removeEventListener("x", null, true), undefined); - // @ts-ignore assertEquals(document.removeEventListener("x", null), undefined); }); diff --git a/cli/tests/unit/fetch_test.ts b/cli/tests/unit/fetch_test.ts index 37fca21123830c..98ff42737768a3 100644 --- a/cli/tests/unit/fetch_test.ts +++ b/cli/tests/unit/fetch_test.ts @@ -3,7 +3,7 @@ import { unitTest, assert, assertEquals, - assertStrContains, + assertStringContains, assertThrows, fail, } from "./test_util.ts"; @@ -18,7 +18,7 @@ unitTest({ perms: { net: true } }, async function fetchProtocolError(): Promise< err = err_; } assert(err instanceof TypeError); - assertStrContains(err.message, "not supported"); + assertStringContains(err.message, "not supported"); }); unitTest( @@ -31,7 +31,7 @@ unitTest( err = err_; } assert(err instanceof Deno.errors.Http); - assertStrContains(err.message, "error trying to connect"); + assertStringContains(err.message, "error trying to connect"); } ); @@ -92,31 +92,103 @@ unitTest({ perms: { net: true } }, async function fetchBodyUsed(): Promise< const response = await fetch("http://localhost:4545/cli/tests/fixture.json"); assertEquals(response.bodyUsed, false); assertThrows((): void => { - // Assigning to read-only property throws in the strict mode. - // @ts-ignore - response.bodyUsed = true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (response as any).bodyUsed = true; }); await response.blob(); assertEquals(response.bodyUsed, true); }); -// TODO(ry) response.body shouldn't be iterable. Instead we should use -// response.body.getReader(). -/* +unitTest( + { perms: { net: true } }, + async function fetchBodyUsedReader(): Promise { + const response = await fetch( + "http://localhost:4545/cli/tests/fixture.json" + ); + assert(response.body !== null); + + const reader = response.body.getReader(); + // Getting a reader should lock the stream but does not consume the body + // so bodyUsed should not be true + assertEquals(response.bodyUsed, false); + reader.releaseLock(); + await response.json(); + assertEquals(response.bodyUsed, true); + } +); + +unitTest( + { perms: { net: true } }, + async function fetchBodyUsedCancelStream(): Promise { + const response = await fetch( + "http://localhost:4545/cli/tests/fixture.json" + ); + assert(response.body !== null); + + assertEquals(response.bodyUsed, false); + const promise = response.body.cancel(); + assertEquals(response.bodyUsed, true); + await promise; + } +); + unitTest({ perms: { net: true } }, async function fetchAsyncIterator(): Promise< void > { const response = await fetch("http://localhost:4545/cli/tests/fixture.json"); const headers = response.headers; + + assert(response.body !== null); let total = 0; for await (const chunk of response.body) { total += chunk.length; } assertEquals(total, Number(headers.get("Content-Length"))); - const _json = await response.json(); }); -*/ + +unitTest({ perms: { net: true } }, async function fetchBodyReader(): Promise< + void +> { + const response = await fetch("http://localhost:4545/cli/tests/fixture.json"); + const headers = response.headers; + assert(response.body !== null); + const reader = await response.body.getReader(); + let total = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + assert(value); + total += value.length; + } + + assertEquals(total, Number(headers.get("Content-Length"))); +}); + +unitTest( + { perms: { net: true } }, + async function fetchBodyReaderBigBody(): Promise { + const data = "a".repeat(10 << 10); // 10mb + const response = await fetch( + "http://localhost:4545/cli/tests/echo_server", + { + method: "POST", + body: data, + } + ); + assert(response.body !== null); + const reader = await response.body.getReader(); + let total = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + assert(value); + total += value.length; + } + + assertEquals(total, data.length); + } +); unitTest({ perms: { net: true } }, async function responseClone(): Promise< void @@ -155,11 +227,10 @@ unitTest( assert(formData.has("field_1")); assertEquals(formData.get("field_1")!.toString(), "value_1 \r\n"); assert(formData.has("field_2")); - /* TODO(ry) Re-enable this test once we bring back the global File type. - const file = formData.get("field_2") as File; - assertEquals(file.name, "file.js"); - */ - // Currently we cannot read from file... + const file = formData.get("field_2") as File; + assertEquals(file.name, "file.js"); + + assertEquals(await file.text(), `console.log("Hi")`); } ); @@ -177,6 +248,79 @@ unitTest( } ); +unitTest( + { perms: { net: true } }, + async function fetchInitFormDataBinaryFileBody(): Promise { + // Some random bytes + // prettier-ignore + const binaryFile = new Uint8Array([108,2,0,0,145,22,162,61,157,227,166,77,138,75,180,56,119,188,177,183]); + const response = await fetch("http://localhost:4545/echo_multipart_file", { + method: "POST", + body: binaryFile, + }); + const resultForm = await response.formData(); + const resultFile = resultForm.get("file") as File; + + assertEquals(resultFile.type, "application/octet-stream"); + assertEquals(resultFile.name, "file.bin"); + assertEquals(new Uint8Array(await resultFile.arrayBuffer()), binaryFile); + } +); + +unitTest( + { perms: { net: true } }, + async function fetchInitFormDataMultipleFilesBody(): Promise { + const files = [ + { + // prettier-ignore + content: new Uint8Array([137,80,78,71,13,10,26,10, 137, 1, 25]), + type: "image/png", + name: "image", + fileName: "some-image.png", + }, + { + // prettier-ignore + content: new Uint8Array([108,2,0,0,145,22,162,61,157,227,166,77,138,75,180,56,119,188,177,183]), + name: "file", + fileName: "file.bin", + expectedType: "application/octet-stream", + }, + { + content: new TextEncoder().encode("deno land"), + type: "text/plain", + name: "text", + fileName: "deno.txt", + }, + ]; + const form = new FormData(); + form.append("field", "value"); + for (const file of files) { + form.append( + file.name, + new Blob([file.content], { type: file.type }), + file.fileName + ); + } + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: form, + }); + const resultForm = await response.formData(); + assertEquals(form.get("field"), resultForm.get("field")); + for (const file of files) { + const inputFile = form.get(file.name) as File; + const resultFile = resultForm.get(file.name) as File; + assertEquals(inputFile.size, resultFile.size); + assertEquals(inputFile.name, resultFile.name); + assertEquals(file.expectedType || file.type, resultFile.type); + assertEquals( + new Uint8Array(await resultFile.arrayBuffer()), + file.content + ); + } + } +); + unitTest( { perms: { net: true }, @@ -206,14 +350,13 @@ unitTest( unitTest( { - // FIXME(bartlomieju): - // The feature below is not implemented, but the test should work after implementation - ignore: true, perms: { net: true }, }, async function fetchWithInfRedirection(): Promise { const response = await fetch("http://localhost:4549/cli/tests"); // will redirect to the same place assertEquals(response.status, 0); // network error + assertEquals(response.type, "error"); + assertEquals(response.ok, false); } ); @@ -258,6 +401,19 @@ unitTest( } ); +unitTest( + { perms: { net: true } }, + async function fetchInitArrayBufferBody(): Promise { + const data = "Hello World"; + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: new TextEncoder().encode(data).buffer, + }); + const text = await response.text(); + assertEquals(text, data); + } +); + unitTest( { perms: { net: true } }, async function fetchInitURLSearchParamsBody(): Promise { @@ -307,6 +463,54 @@ unitTest( } ); +unitTest( + { perms: { net: true } }, + async function fetchInitFormDataBlobFilenameBody(): Promise { + const form = new FormData(); + form.append("field", "value"); + form.append("file", new Blob([new TextEncoder().encode("deno")])); + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: form, + }); + const resultForm = await response.formData(); + assertEquals(form.get("field"), resultForm.get("field")); + const file = resultForm.get("file"); + assert(file instanceof File); + assertEquals(file.name, "blob"); + } +); + +unitTest( + { perms: { net: true } }, + async function fetchInitFormDataTextFileBody(): Promise { + const fileContent = "deno land"; + const form = new FormData(); + form.append("field", "value"); + form.append( + "file", + new Blob([new TextEncoder().encode(fileContent)], { + type: "text/plain", + }), + "deno.txt" + ); + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: form, + }); + const resultForm = await response.formData(); + assertEquals(form.get("field"), resultForm.get("field")); + + const file = form.get("file") as File; + const resultFile = resultForm.get("file") as File; + + assertEquals(file.size, resultFile.size); + assertEquals(file.name, resultFile.name); + assertEquals(file.type, resultFile.type); + assertEquals(await file.text(), await resultFile.text()); + } +); + unitTest({ perms: { net: true } }, async function fetchUserAgent(): Promise< void > { @@ -367,8 +571,6 @@ function bufferServer(addr: string): Deno.Buffer { unitTest( { - // FIXME(bartlomieju) - ignore: true, perms: { net: true }, }, async function fetchRequest(): Promise { @@ -381,6 +583,7 @@ unitTest( ["Foo", "Bar"], ], }); + await response.arrayBuffer(); assertEquals(response.status, 404); assertEquals(response.headers.get("Content-Length"), "2"); @@ -389,6 +592,9 @@ unitTest( "POST /blah HTTP/1.1\r\n", "hello: World\r\n", "foo: Bar\r\n", + "accept: */*\r\n", + `user-agent: Deno/${Deno.version.deno}\r\n`, + "accept-encoding: gzip, br\r\n", `host: ${addr}\r\n\r\n`, ].join(""); assertEquals(actual, expected); @@ -397,8 +603,6 @@ unitTest( unitTest( { - // FIXME(bartlomieju) - ignore: true, perms: { net: true }, }, async function fetchPostBodyString(): Promise { @@ -413,6 +617,7 @@ unitTest( ], body, }); + await response.arrayBuffer(); assertEquals(response.status, 404); assertEquals(response.headers.get("Content-Length"), "2"); @@ -421,6 +626,10 @@ unitTest( "POST /blah HTTP/1.1\r\n", "hello: World\r\n", "foo: Bar\r\n", + "content-type: text/plain;charset=UTF-8\r\n", + "accept: */*\r\n", + `user-agent: Deno/${Deno.version.deno}\r\n`, + "accept-encoding: gzip, br\r\n", `host: ${addr}\r\n`, `content-length: ${body.length}\r\n\r\n`, body, @@ -431,8 +640,6 @@ unitTest( unitTest( { - // FIXME(bartlomieju) - ignore: true, perms: { net: true }, }, async function fetchPostBodyTypedArray(): Promise { @@ -448,6 +655,7 @@ unitTest( ], body, }); + await response.arrayBuffer(); assertEquals(response.status, 404); assertEquals(response.headers.get("Content-Length"), "2"); @@ -456,6 +664,9 @@ unitTest( "POST /blah HTTP/1.1\r\n", "hello: World\r\n", "foo: Bar\r\n", + "accept: */*\r\n", + `user-agent: Deno/${Deno.version.deno}\r\n`, + "accept-encoding: gzip, br\r\n", `host: ${addr}\r\n`, `content-length: ${body.byteLength}\r\n\r\n`, bodyStr, @@ -519,16 +730,178 @@ unitTest(function responseRedirect(): void { assertEquals(redir.type, "default"); }); -unitTest(function responseConstructionHeaderRemoval(): void { - const res = new Response( - "example.com", - 200, - "OK", - [["Set-Cookie", "mysessionid"]], - -1, - false, - "basic", - null - ); - assert(res.headers.get("Set-Cookie") != "mysessionid"); +unitTest({ perms: { net: true } }, async function fetchBodyReadTwice(): Promise< + void +> { + const response = await fetch("http://localhost:4545/cli/tests/fixture.json"); + + // Read body + const _json = await response.json(); + assert(_json); + + // All calls after the body was consumed, should fail + const methods = ["json", "text", "formData", "arrayBuffer"] as const; + for (const method of methods) { + try { + await response[method](); + fail( + "Reading body multiple times should failed, the stream should've been locked." + ); + } catch {} + } }); + +unitTest( + { perms: { net: true } }, + async function fetchBodyReaderAfterRead(): Promise { + const response = await fetch( + "http://localhost:4545/cli/tests/fixture.json" + ); + assert(response.body !== null); + const reader = await response.body.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + assert(value); + } + + try { + response.body.getReader(); + fail("The stream should've been locked."); + } catch {} + } +); + +unitTest( + { perms: { net: true } }, + async function fetchBodyReaderWithCancelAndNewReader(): Promise { + const data = "a".repeat(1 << 10); + const response = await fetch( + "http://localhost:4545/cli/tests/echo_server", + { + method: "POST", + body: data, + } + ); + assert(response.body !== null); + const firstReader = await response.body.getReader(); + + // Acquire reader without reading & release + await firstReader.releaseLock(); + + const reader = await response.body.getReader(); + + let total = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + assert(value); + total += value.length; + } + + assertEquals(total, data.length); + } +); + +unitTest( + { perms: { net: true } }, + async function fetchBodyReaderWithReadCancelAndNewReader(): Promise { + const data = "a".repeat(1 << 10); + + const response = await fetch( + "http://localhost:4545/cli/tests/echo_server", + { + method: "POST", + body: data, + } + ); + assert(response.body !== null); + const firstReader = await response.body.getReader(); + + // Do one single read with first reader + const { value: firstValue } = await firstReader.read(); + assert(firstValue); + await firstReader.releaseLock(); + + // Continue read with second reader + const reader = await response.body.getReader(); + let total = firstValue.length || 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + assert(value); + total += value.length; + } + assertEquals(total, data.length); + } +); + +unitTest( + { perms: { net: true } }, + async function fetchResourceCloseAfterStreamCancel(): Promise { + const res = await fetch("http://localhost:4545/cli/tests/fixture.json"); + assert(res.body !== null); + + // After ReadableStream.cancel is called, resource handle must be closed + // The test should not fail with: Test case is leaking resources + await res.body.cancel(); + } +); + +unitTest( + { perms: { net: true } }, + async function fetchNullBodyStatus(): Promise { + const nullBodyStatus = [101, 204, 205, 304]; + + for (const status of nullBodyStatus) { + const headers = new Headers([["x-status", String(status)]]); + const res = await fetch("http://localhost:4545/cli/tests/echo_server", { + body: "deno", + method: "POST", + headers, + }); + assertEquals(res.body, null); + assertEquals(res.status, status); + } + } +); + +unitTest( + { perms: { net: true } }, + function fetchResponseConstructorNullBody(): void { + const nullBodyStatus = [204, 205, 304]; + + for (const status of nullBodyStatus) { + try { + new Response("deno", { status }); + fail("Response with null body status cannot have body"); + } catch (e) { + assert(e instanceof TypeError); + assertEquals( + e.message, + "Response with null body status cannot have body" + ); + } + } + } +); + +unitTest( + { perms: { net: true } }, + function fetchResponseConstructorInvalidStatus(): void { + const invalidStatus = [101, 600, 199]; + + for (const status of invalidStatus) { + try { + new Response("deno", { status }); + fail("Invalid status"); + } catch (e) { + assert(e instanceof RangeError); + assertEquals( + e.message, + `The status provided (${status}) is outside the range [200, 599]` + ); + } + } + } +); diff --git a/cli/tests/unit/files_test.ts b/cli/tests/unit/files_test.ts index a035c70742803d..46bcb2173bc16c 100644 --- a/cli/tests/unit/files_test.ts +++ b/cli/tests/unit/files_test.ts @@ -3,7 +3,7 @@ import { unitTest, assert, assertEquals, - assertStrContains, + assertStringContains, } from "./test_util.ts"; unitTest(function filesStdioFileDescriptors(): void { @@ -225,7 +225,7 @@ unitTest(async function openOptions(): Promise { err = e; } assert(!!err); - assertStrContains( + assertStringContains( err.message, "OpenOptions requires at least one option to be true" ); @@ -236,7 +236,10 @@ unitTest(async function openOptions(): Promise { err = e; } assert(!!err); - assertStrContains(err.message, "'truncate' option requires 'write' option"); + assertStringContains( + err.message, + "'truncate' option requires 'write' option" + ); try { await Deno.open(filename, { create: true, write: false }); @@ -244,7 +247,7 @@ unitTest(async function openOptions(): Promise { err = e; } assert(!!err); - assertStrContains( + assertStringContains( err.message, "'create' or 'createNew' options require 'write' or 'append' option" ); @@ -255,7 +258,7 @@ unitTest(async function openOptions(): Promise { err = e; } assert(!!err); - assertStrContains( + assertStringContains( err.message, "'create' or 'createNew' options require 'write' or 'append' option" ); @@ -290,8 +293,8 @@ unitTest( // writing null should throw an error let err; try { - // @ts-ignore - await file.write(null); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await file.write(null as any); } catch (e) { err = e; } @@ -322,8 +325,8 @@ unitTest( // reading file into null buffer should throw an error let err; try { - // @ts-ignore - await file.read(null); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await file.read(null as any); } catch (e) { err = e; } @@ -560,7 +563,7 @@ unitTest({ perms: { read: true } }, async function seekMode(): Promise { } assert(!!err); assert(err instanceof TypeError); - assertStrContains(err.message, "Invalid seek mode"); + assertStringContains(err.message, "Invalid seek mode"); // We should still be able to read the file // since it is still open. diff --git a/cli/tests/unit/form_data_test.ts b/cli/tests/unit/form_data_test.ts index 10cbd30a79561c..338fdd0895fab8 100644 --- a/cli/tests/unit/form_data_test.ts +++ b/cli/tests/unit/form_data_test.ts @@ -3,10 +3,10 @@ import { unitTest, assert, assertEquals, - assertStrContains, + assertStringContains, } from "./test_util.ts"; -unitTest({ ignore: true }, function formDataHasCorrectNameProp(): void { +unitTest(function formDataHasCorrectNameProp(): void { assertEquals(FormData.name, "FormData"); }); @@ -41,10 +41,10 @@ unitTest(function formDataParamsGetSuccess(): void { formData.append("a", "true"); formData.append("b", "false"); formData.append("a", "null"); - // @ts-ignore - formData.append("d", undefined); - // @ts-ignore - formData.append("e", null); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + formData.append("d", undefined as any); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + formData.append("e", null as any); assertEquals(formData.get("a"), "true"); assertEquals(formData.get("b"), "false"); assertEquals(formData.get("c"), null); @@ -70,11 +70,11 @@ unitTest(function formDataParamsSetSuccess(): void { assertEquals(formData.getAll("b"), ["false"]); formData.set("a", "false"); assertEquals(formData.getAll("a"), ["false"]); - // @ts-ignore - formData.set("d", undefined); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + formData.set("d", undefined as any); assertEquals(formData.get("d"), "undefined"); - // @ts-ignore - formData.set("e", null); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + formData.set("e", null as any); assertEquals(formData.get("e"), "null"); }); @@ -99,6 +99,15 @@ unitTest(function formDataSetEmptyBlobSuccess(): void { */ }); +unitTest(function formDataBlobFilename(): void { + const formData = new FormData(); + const content = new TextEncoder().encode("deno"); + formData.set("a", new Blob([content])); + const file = formData.get("a"); + assert(file instanceof File); + assertEquals(file.name, "blob"); +}); + unitTest(function formDataParamsForEachSuccess(): void { const init = [ ["a", "54"], @@ -134,8 +143,8 @@ unitTest(function formDataParamsArgumentsCheck(): void { let hasThrown = 0; let errMsg = ""; try { - // @ts-ignore - formData[method](); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (formData as any)[method](); hasThrown = 1; } catch (err) { errMsg = err.message; @@ -146,7 +155,7 @@ unitTest(function formDataParamsArgumentsCheck(): void { } } assertEquals(hasThrown, 2); - assertStrContains( + assertStringContains( errMsg, `${method} requires at least 1 argument, but only 0 present` ); @@ -158,8 +167,8 @@ unitTest(function formDataParamsArgumentsCheck(): void { let errMsg = ""; try { - // @ts-ignore - formData[method](); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (formData as any)[method](); hasThrown = 1; } catch (err) { errMsg = err.message; @@ -170,7 +179,7 @@ unitTest(function formDataParamsArgumentsCheck(): void { } } assertEquals(hasThrown, 2); - assertStrContains( + assertStringContains( errMsg, `${method} requires at least 2 arguments, but only 0 present` ); @@ -178,8 +187,8 @@ unitTest(function formDataParamsArgumentsCheck(): void { hasThrown = 0; errMsg = ""; try { - // @ts-ignore - formData[method]("foo"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (formData as any)[method]("foo"); hasThrown = 1; } catch (err) { errMsg = err.message; @@ -190,7 +199,7 @@ unitTest(function formDataParamsArgumentsCheck(): void { } } assertEquals(hasThrown, 2); - assertStrContains( + assertStringContains( errMsg, `${method} requires at least 2 arguments, but only 1 present` ); diff --git a/cli/tests/unit/get_random_values_test.ts b/cli/tests/unit/get_random_values_test.ts index 76fa732ea5c11d..69df0d44b3819b 100644 --- a/cli/tests/unit/get_random_values_test.ts +++ b/cli/tests/unit/get_random_values_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { unitTest, assertNotEquals, assertStrictEq } from "./test_util.ts"; +import { unitTest, assertNotEquals, assertStrictEquals } from "./test_util.ts"; unitTest(function getRandomValuesInt8Array(): void { const arr = new Int8Array(32); @@ -47,5 +47,5 @@ unitTest(function getRandomValuesReturnValue(): void { const arr = new Uint32Array(8); const rtn = crypto.getRandomValues(arr); assertNotEquals(arr, new Uint32Array(8)); - assertStrictEq(rtn, arr); + assertStrictEquals(rtn, arr); }); diff --git a/cli/tests/unit/globals_test.ts b/cli/tests/unit/globals_test.ts index aa8b4f46e7fe53..ccea6e74c36f3e 100644 --- a/cli/tests/unit/globals_test.ts +++ b/cli/tests/unit/globals_test.ts @@ -45,16 +45,24 @@ unitTest(function webAssemblyExists(): void { assert(typeof WebAssembly.compile === "function"); }); +/* eslint-disable @typescript-eslint/no-namespace, @typescript-eslint/no-explicit-any,no-var */ +declare global { + namespace Deno { + var core: any; + } +} +/* eslint-enable */ + unitTest(function DenoNamespaceImmutable(): void { const denoCopy = window.Deno; try { - // @ts-ignore - Deno = 1; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (Deno as any) = 1; } catch {} assert(denoCopy === Deno); try { - // @ts-ignore - window.Deno = 1; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (window as any).Deno = 1; } catch {} assert(denoCopy === Deno); try { @@ -64,8 +72,8 @@ unitTest(function DenoNamespaceImmutable(): void { const { readFile } = Deno; try { - // @ts-ignore - Deno.readFile = 1; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (Deno as any).readFile = 1; } catch {} assert(readFile === Deno.readFile); try { @@ -73,19 +81,14 @@ unitTest(function DenoNamespaceImmutable(): void { } catch {} assert(readFile === Deno.readFile); - // @ts-ignore const { print } = Deno.core; try { - // @ts-ignore Deno.core.print = 1; } catch {} - // @ts-ignore assert(print === Deno.core.print); try { - // @ts-ignore delete Deno.core.print; } catch {} - // @ts-ignore assert(print === Deno.core.print); }); diff --git a/cli/tests/unit/headers_test.ts b/cli/tests/unit/headers_test.ts index aaa8298379dcb8..8fbf1d4e40d566 100644 --- a/cli/tests/unit/headers_test.ts +++ b/cli/tests/unit/headers_test.ts @@ -3,13 +3,17 @@ import { unitTest, assert, assertEquals, - assertStrContains, + assertStringContains, } from "./test_util.ts"; const { stringifyArgs, - // @ts-ignore TypeScript (as of 3.7) does not support indexing namespaces by symbol + // @ts-expect-error TypeScript (as of 3.7) does not support indexing namespaces by symbol } = Deno[Deno.internal]; +unitTest(function headersHasCorrectNameProp(): void { + assertEquals(Headers.name, "Headers"); +}); + // Logic heavily copied from web-platform-tests, make // sure pass mostly header basic test // ref: https://github.com/web-platform-tests/wpt/blob/7c50c216081d6ea3c9afe553ee7b64534020a1b2/fetch/api/headers/headers-basic.html @@ -18,8 +22,8 @@ unitTest(function newHeaderTest(): void { new Headers(undefined); new Headers({}); try { - // @ts-ignore - new Headers(null); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new Headers(null as any); } catch (e) { assertEquals( e.message, @@ -32,8 +36,8 @@ const headerDict: Record = { name1: "value1", name2: "value2", name3: "value3", - // @ts-ignore - name4: undefined, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + name4: undefined as any, "Content-Type": "value4", }; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -260,17 +264,17 @@ unitTest(function headerParamsShouldThrowTypeError(): void { }); unitTest(function headerParamsArgumentsCheck(): void { - const methodRequireOneParam = ["delete", "get", "has", "forEach"]; + const methodRequireOneParam = ["delete", "get", "has", "forEach"] as const; - const methodRequireTwoParams = ["append", "set"]; + const methodRequireTwoParams = ["append", "set"] as const; methodRequireOneParam.forEach((method): void => { const headers = new Headers(); let hasThrown = 0; let errMsg = ""; try { - // @ts-ignore - headers[method](); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (headers as any)[method](); hasThrown = 1; } catch (err) { errMsg = err.message; @@ -281,7 +285,7 @@ unitTest(function headerParamsArgumentsCheck(): void { } } assertEquals(hasThrown, 2); - assertStrContains( + assertStringContains( errMsg, `${method} requires at least 1 argument, but only 0 present` ); @@ -293,8 +297,8 @@ unitTest(function headerParamsArgumentsCheck(): void { let errMsg = ""; try { - // @ts-ignore - headers[method](); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (headers as any)[method](); hasThrown = 1; } catch (err) { errMsg = err.message; @@ -305,7 +309,7 @@ unitTest(function headerParamsArgumentsCheck(): void { } } assertEquals(hasThrown, 2); - assertStrContains( + assertStringContains( errMsg, `${method} requires at least 2 arguments, but only 0 present` ); @@ -313,8 +317,8 @@ unitTest(function headerParamsArgumentsCheck(): void { hasThrown = 0; errMsg = ""; try { - // @ts-ignore - headers[method]("foo"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (headers as any)[method]("foo"); hasThrown = 1; } catch (err) { errMsg = err.message; @@ -325,7 +329,7 @@ unitTest(function headerParamsArgumentsCheck(): void { } } assertEquals(hasThrown, 2); - assertStrContains( + assertStringContains( errMsg, `${method} requires at least 2 arguments, but only 1 present` ); diff --git a/cli/tests/unit/internals_test.ts b/cli/tests/unit/internals_test.ts index abd4c94c3d4945..3f4bdae79e51bc 100644 --- a/cli/tests/unit/internals_test.ts +++ b/cli/tests/unit/internals_test.ts @@ -4,7 +4,7 @@ import { unitTest, assert } from "./test_util.ts"; unitTest(function internalsExists(): void { const { stringifyArgs, - // @ts-ignore TypeScript (as of 3.7) does not support indexing namespaces by symbol + // @ts-expect-error TypeScript (as of 3.7) does not support indexing namespaces by symbol } = Deno[Deno.internal]; assert(!!stringifyArgs); }); diff --git a/cli/tests/unit/net_test.ts b/cli/tests/unit/net_test.ts index 9e9a1e5e8f111b..4fd253a6c90a0b 100644 --- a/cli/tests/unit/net_test.ts +++ b/cli/tests/unit/net_test.ts @@ -4,6 +4,7 @@ import { assert, assertEquals, createResolvable, + assertNotEquals, } from "./test_util.ts"; unitTest({ perms: { net: true } }, function netTcpListenClose(): void { @@ -11,6 +12,7 @@ unitTest({ perms: { net: true } }, function netTcpListenClose(): void { assert(listener.addr.transport === "tcp"); assertEquals(listener.addr.hostname, "127.0.0.1"); assertEquals(listener.addr.port, 3500); + assertNotEquals(listener.rid, 0); listener.close(); }); diff --git a/cli/tests/unit/os_test.ts b/cli/tests/unit/os_test.ts index e9900253431c32..72c88b0e10e253 100644 --- a/cli/tests/unit/os_test.ts +++ b/cli/tests/unit/os_test.ts @@ -277,15 +277,13 @@ unitTest({ perms: { read: true } }, function execPath(): void { }); unitTest({ perms: { read: false } }, function execPathPerm(): void { - let caughtError = false; - try { - Deno.execPath(); - } catch (err) { - caughtError = true; - assert(err instanceof Deno.errors.PermissionDenied); - assertEquals(err.name, "PermissionDenied"); - } - assert(caughtError); + assertThrows( + () => { + Deno.execPath(); + }, + Deno.errors.PermissionDenied, + "read access to , run again with the --allow-read flag" + ); }); unitTest({ perms: { env: true } }, function loadavgSuccess(): void { diff --git a/cli/tests/unit/process_test.ts b/cli/tests/unit/process_test.ts index 1ea6f95b704f16..abd84504111df7 100644 --- a/cli/tests/unit/process_test.ts +++ b/cli/tests/unit/process_test.ts @@ -2,7 +2,7 @@ import { assert, assertEquals, - assertStrContains, + assertStringContains, unitTest, } from "./test_util.ts"; const { @@ -260,8 +260,8 @@ unitTest( const decoder = new TextDecoder(); const text = decoder.decode(fileContents); - assertStrContains(text, "error"); - assertStrContains(text, "output"); + assertStringContains(text, "error"); + assertStringContains(text, "output"); } ); diff --git a/cli/tests/unit/remove_test.ts b/cli/tests/unit/remove_test.ts index 35e5c821e172c3..f9095c1c1da8a3 100644 --- a/cli/tests/unit/remove_test.ts +++ b/cli/tests/unit/remove_test.ts @@ -1,465 +1,284 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import { unitTest, assert, assertEquals } from "./test_util.ts"; -// SYNC +const REMOVE_METHODS = ["remove", "removeSync"] as const; unitTest( { perms: { write: true, read: true } }, - function removeSyncDirSuccess(): void { - // REMOVE EMPTY DIRECTORY - const path = Deno.makeTempDirSync() + "/subdir"; - Deno.mkdirSync(path); - const pathInfo = Deno.statSync(path); - assert(pathInfo.isDirectory); // check exist first - Deno.removeSync(path); // remove - // We then check again after remove - let err; - try { - Deno.statSync(path); - } catch (e) { - err = e; - } - // Directory is gone - assert(err instanceof Deno.errors.NotFound); - } -); - -unitTest( - { perms: { write: true, read: true } }, - function removeSyncFileSuccess(): void { - // REMOVE FILE - const enc = new TextEncoder(); - const data = enc.encode("Hello"); - const filename = Deno.makeTempDirSync() + "/test.txt"; - Deno.writeFileSync(filename, data, { mode: 0o666 }); - const fileInfo = Deno.statSync(filename); - assert(fileInfo.isFile); // check exist first - Deno.removeSync(filename); // remove - // We then check again after remove - let err; - try { - Deno.statSync(filename); - } catch (e) { - err = e; + async function removeDirSuccess(): Promise { + for (const method of REMOVE_METHODS) { + // REMOVE EMPTY DIRECTORY + const path = Deno.makeTempDirSync() + "/subdir"; + Deno.mkdirSync(path); + const pathInfo = Deno.statSync(path); + assert(pathInfo.isDirectory); // check exist first + await Deno[method](path); // remove + // We then check again after remove + let err; + try { + Deno.statSync(path); + } catch (e) { + err = e; + } + // Directory is gone + assert(err instanceof Deno.errors.NotFound); } - // File is gone - assert(err instanceof Deno.errors.NotFound); } ); unitTest( { perms: { write: true, read: true } }, - function removeSyncFail(): void { - // NON-EMPTY DIRECTORY - const path = Deno.makeTempDirSync() + "/dir/subdir"; - const subPath = path + "/subsubdir"; - Deno.mkdirSync(path, { recursive: true }); - Deno.mkdirSync(subPath); - const pathInfo = Deno.statSync(path); - assert(pathInfo.isDirectory); // check exist first - const subPathInfo = Deno.statSync(subPath); - assert(subPathInfo.isDirectory); // check exist first - let err; - try { - // Should not be able to recursively remove - Deno.removeSync(path); - } catch (e) { - err = e; - } - // TODO(ry) Is Other really the error we should get here? What would Go do? - assert(err instanceof Error); - // NON-EXISTENT DIRECTORY/FILE - try { - // Non-existent - Deno.removeSync("/baddir"); - } catch (e) { - err = e; + async function removeFileSuccess(): Promise { + for (const method of REMOVE_METHODS) { + // REMOVE FILE + const enc = new TextEncoder(); + const data = enc.encode("Hello"); + const filename = Deno.makeTempDirSync() + "/test.txt"; + Deno.writeFileSync(filename, data, { mode: 0o666 }); + const fileInfo = Deno.statSync(filename); + assert(fileInfo.isFile); // check exist first + await Deno[method](filename); // remove + // We then check again after remove + let err; + try { + Deno.statSync(filename); + } catch (e) { + err = e; + } + // File is gone + assert(err instanceof Deno.errors.NotFound); } - assert(err instanceof Deno.errors.NotFound); } ); unitTest( { perms: { write: true, read: true } }, - function removeSyncDanglingSymlinkSuccess(): void { - const danglingSymlinkPath = Deno.makeTempDirSync() + "/dangling_symlink"; - if (Deno.build.os === "windows") { - Deno.symlinkSync("unexistent_file", danglingSymlinkPath, { - type: "file", - }); - } else { - Deno.symlinkSync("unexistent_file", danglingSymlinkPath); - } - const pathInfo = Deno.lstatSync(danglingSymlinkPath); - assert(pathInfo.isSymlink); - Deno.removeSync(danglingSymlinkPath); - let err; - try { - Deno.lstatSync(danglingSymlinkPath); - } catch (e) { - err = e; + async function removeFail(): Promise { + for (const method of REMOVE_METHODS) { + // NON-EMPTY DIRECTORY + const path = Deno.makeTempDirSync() + "/dir/subdir"; + const subPath = path + "/subsubdir"; + Deno.mkdirSync(path, { recursive: true }); + Deno.mkdirSync(subPath); + const pathInfo = Deno.statSync(path); + assert(pathInfo.isDirectory); // check exist first + const subPathInfo = Deno.statSync(subPath); + assert(subPathInfo.isDirectory); // check exist first + let err; + try { + // Should not be able to recursively remove + await Deno[method](path); + } catch (e) { + err = e; + } + // TODO(ry) Is Other really the error we should get here? What would Go do? + assert(err instanceof Error); + // NON-EXISTENT DIRECTORY/FILE + try { + // Non-existent + await Deno[method]("/baddir"); + } catch (e) { + err = e; + } + assert(err instanceof Deno.errors.NotFound); } - assert(err instanceof Deno.errors.NotFound); } ); unitTest( { perms: { write: true, read: true } }, - function removeSyncValidSymlinkSuccess(): void { - const encoder = new TextEncoder(); - const data = encoder.encode("Test"); - const tempDir = Deno.makeTempDirSync(); - const filePath = tempDir + "/test.txt"; - const validSymlinkPath = tempDir + "/valid_symlink"; - Deno.writeFileSync(filePath, data, { mode: 0o666 }); - if (Deno.build.os === "windows") { - Deno.symlinkSync(filePath, validSymlinkPath, { type: "file" }); - } else { - Deno.symlinkSync(filePath, validSymlinkPath); - } - const symlinkPathInfo = Deno.statSync(validSymlinkPath); - assert(symlinkPathInfo.isFile); - Deno.removeSync(validSymlinkPath); - let err; - try { - Deno.statSync(validSymlinkPath); - } catch (e) { - err = e; + async function removeDanglingSymlinkSuccess(): Promise { + for (const method of REMOVE_METHODS) { + const danglingSymlinkPath = Deno.makeTempDirSync() + "/dangling_symlink"; + if (Deno.build.os === "windows") { + Deno.symlinkSync("unexistent_file", danglingSymlinkPath, { + type: "file", + }); + } else { + Deno.symlinkSync("unexistent_file", danglingSymlinkPath); + } + const pathInfo = Deno.lstatSync(danglingSymlinkPath); + assert(pathInfo.isSymlink); + await Deno[method](danglingSymlinkPath); + let err; + try { + Deno.lstatSync(danglingSymlinkPath); + } catch (e) { + err = e; + } + assert(err instanceof Deno.errors.NotFound); } - Deno.removeSync(filePath); - assert(err instanceof Deno.errors.NotFound); } ); -unitTest({ perms: { write: false } }, function removeSyncPerm(): void { - let err; - try { - Deno.removeSync("/baddir"); - } catch (e) { - err = e; - } - assert(err instanceof Deno.errors.PermissionDenied); - assertEquals(err.name, "PermissionDenied"); -}); - unitTest( { perms: { write: true, read: true } }, - function removeAllSyncDirSuccess(): void { - // REMOVE EMPTY DIRECTORY - let path = Deno.makeTempDirSync() + "/dir/subdir"; - Deno.mkdirSync(path, { recursive: true }); - let pathInfo = Deno.statSync(path); - assert(pathInfo.isDirectory); // check exist first - Deno.removeSync(path, { recursive: true }); // remove - // We then check again after remove - let err; - try { - Deno.statSync(path); - } catch (e) { - err = e; - } - // Directory is gone - assert(err instanceof Deno.errors.NotFound); - - // REMOVE NON-EMPTY DIRECTORY - path = Deno.makeTempDirSync() + "/dir/subdir"; - const subPath = path + "/subsubdir"; - Deno.mkdirSync(path, { recursive: true }); - Deno.mkdirSync(subPath); - pathInfo = Deno.statSync(path); - assert(pathInfo.isDirectory); // check exist first - const subPathInfo = Deno.statSync(subPath); - assert(subPathInfo.isDirectory); // check exist first - Deno.removeSync(path, { recursive: true }); // remove - // We then check parent directory again after remove - try { - Deno.statSync(path); - } catch (e) { - err = e; + async function removeValidSymlinkSuccess(): Promise { + for (const method of REMOVE_METHODS) { + const encoder = new TextEncoder(); + const data = encoder.encode("Test"); + const tempDir = Deno.makeTempDirSync(); + const filePath = tempDir + "/test.txt"; + const validSymlinkPath = tempDir + "/valid_symlink"; + Deno.writeFileSync(filePath, data, { mode: 0o666 }); + if (Deno.build.os === "windows") { + Deno.symlinkSync(filePath, validSymlinkPath, { type: "file" }); + } else { + Deno.symlinkSync(filePath, validSymlinkPath); + } + const symlinkPathInfo = Deno.statSync(validSymlinkPath); + assert(symlinkPathInfo.isFile); + await Deno[method](validSymlinkPath); + let err; + try { + Deno.statSync(validSymlinkPath); + } catch (e) { + err = e; + } + await Deno[method](filePath); + assert(err instanceof Deno.errors.NotFound); } - // Directory is gone - assert(err instanceof Deno.errors.NotFound); } ); -unitTest( - { perms: { write: true, read: true } }, - function removeAllSyncFileSuccess(): void { - // REMOVE FILE - const enc = new TextEncoder(); - const data = enc.encode("Hello"); - const filename = Deno.makeTempDirSync() + "/test.txt"; - Deno.writeFileSync(filename, data, { mode: 0o666 }); - const fileInfo = Deno.statSync(filename); - assert(fileInfo.isFile); // check exist first - Deno.removeSync(filename, { recursive: true }); // remove - // We then check again after remove +unitTest({ perms: { write: false } }, async function removePerm(): Promise< + void +> { + for (const method of REMOVE_METHODS) { let err; try { - Deno.statSync(filename); + await Deno[method]("/baddir"); } catch (e) { err = e; } - // File is gone - assert(err instanceof Deno.errors.NotFound); - } -); - -unitTest({ perms: { write: true } }, function removeAllSyncFail(): void { - // NON-EXISTENT DIRECTORY/FILE - let err; - try { - // Non-existent - Deno.removeSync("/baddir", { recursive: true }); - } catch (e) { - err = e; - } - assert(err instanceof Deno.errors.NotFound); -}); - -unitTest({ perms: { write: false } }, function removeAllSyncPerm(): void { - let err; - try { - Deno.removeSync("/baddir", { recursive: true }); - } catch (e) { - err = e; + assert(err instanceof Deno.errors.PermissionDenied); + assertEquals(err.name, "PermissionDenied"); } - assert(err instanceof Deno.errors.PermissionDenied); - assertEquals(err.name, "PermissionDenied"); }); -// ASYNC - -unitTest( - { perms: { write: true, read: true } }, - async function removeDirSuccess(): Promise { - // REMOVE EMPTY DIRECTORY - const path = Deno.makeTempDirSync() + "/dir/subdir"; - Deno.mkdirSync(path, { recursive: true }); - const pathInfo = Deno.statSync(path); - assert(pathInfo.isDirectory); // check exist first - await Deno.remove(path); // remove - // We then check again after remove - let err; - try { - Deno.statSync(path); - } catch (e) { - err = e; - } - // Directory is gone - assert(err instanceof Deno.errors.NotFound); - } -); - unitTest( { perms: { write: true, read: true } }, - async function removeFileSuccess(): Promise { - // REMOVE FILE - const enc = new TextEncoder(); - const data = enc.encode("Hello"); - const filename = Deno.makeTempDirSync() + "/test.txt"; - Deno.writeFileSync(filename, data, { mode: 0o666 }); - const fileInfo = Deno.statSync(filename); - assert(fileInfo.isFile); // check exist first - await Deno.remove(filename); // remove - // We then check again after remove - let err; - try { - Deno.statSync(filename); - } catch (e) { - err = e; - } - // File is gone - assert(err instanceof Deno.errors.NotFound); - } -); + async function removeAllDirSuccess(): Promise { + for (const method of REMOVE_METHODS) { + // REMOVE EMPTY DIRECTORY + let path = Deno.makeTempDirSync() + "/dir/subdir"; + Deno.mkdirSync(path, { recursive: true }); + let pathInfo = Deno.statSync(path); + assert(pathInfo.isDirectory); // check exist first + await Deno[method](path, { recursive: true }); // remove + // We then check again after remove + let err; + try { + Deno.statSync(path); + } catch (e) { + err = e; + } + // Directory is gone + assert(err instanceof Deno.errors.NotFound); -unitTest( - { perms: { write: true, read: true } }, - async function removeFail(): Promise { - // NON-EMPTY DIRECTORY - const path = Deno.makeTempDirSync() + "/dir/subdir"; - const subPath = path + "/subsubdir"; - Deno.mkdirSync(path, { recursive: true }); - Deno.mkdirSync(subPath); - const pathInfo = Deno.statSync(path); - assert(pathInfo.isDirectory); // check exist first - const subPathInfo = Deno.statSync(subPath); - assert(subPathInfo.isDirectory); // check exist first - let err; - try { - // Should not be able to recursively remove - await Deno.remove(path); - } catch (e) { - err = e; - } - assert(err instanceof Error); - // NON-EXISTENT DIRECTORY/FILE - try { - // Non-existent - await Deno.remove("/baddir"); - } catch (e) { - err = e; + // REMOVE NON-EMPTY DIRECTORY + path = Deno.makeTempDirSync() + "/dir/subdir"; + const subPath = path + "/subsubdir"; + Deno.mkdirSync(path, { recursive: true }); + Deno.mkdirSync(subPath); + pathInfo = Deno.statSync(path); + assert(pathInfo.isDirectory); // check exist first + const subPathInfo = Deno.statSync(subPath); + assert(subPathInfo.isDirectory); // check exist first + await Deno[method](path, { recursive: true }); // remove + // We then check parent directory again after remove + try { + Deno.statSync(path); + } catch (e) { + err = e; + } + // Directory is gone + assert(err instanceof Deno.errors.NotFound); } - assert(err instanceof Deno.errors.NotFound); } ); unitTest( { perms: { write: true, read: true } }, - async function removeDanglingSymlinkSuccess(): Promise { - const danglingSymlinkPath = Deno.makeTempDirSync() + "/dangling_symlink"; - if (Deno.build.os === "windows") { - Deno.symlinkSync("unexistent_file", danglingSymlinkPath, { - type: "file", - }); - } else { - Deno.symlinkSync("unexistent_file", danglingSymlinkPath); - } - const pathInfo = Deno.lstatSync(danglingSymlinkPath); - assert(pathInfo.isSymlink); - await Deno.remove(danglingSymlinkPath); - let err; - try { - Deno.lstatSync(danglingSymlinkPath); - } catch (e) { - err = e; + async function removeAllFileSuccess(): Promise { + for (const method of REMOVE_METHODS) { + // REMOVE FILE + const enc = new TextEncoder(); + const data = enc.encode("Hello"); + const filename = Deno.makeTempDirSync() + "/test.txt"; + Deno.writeFileSync(filename, data, { mode: 0o666 }); + const fileInfo = Deno.statSync(filename); + assert(fileInfo.isFile); // check exist first + await Deno[method](filename, { recursive: true }); // remove + // We then check again after remove + let err; + try { + Deno.statSync(filename); + } catch (e) { + err = e; + } + // File is gone + assert(err instanceof Deno.errors.NotFound); } - assert(err instanceof Deno.errors.NotFound); } ); -unitTest( - { perms: { write: true, read: true } }, - async function removeValidSymlinkSuccess(): Promise { - const encoder = new TextEncoder(); - const data = encoder.encode("Test"); - const tempDir = Deno.makeTempDirSync(); - const filePath = tempDir + "/test.txt"; - const validSymlinkPath = tempDir + "/valid_symlink"; - Deno.writeFileSync(filePath, data, { mode: 0o666 }); - if (Deno.build.os === "windows") { - Deno.symlinkSync(filePath, validSymlinkPath, { type: "file" }); - } else { - Deno.symlinkSync(filePath, validSymlinkPath); - } - const symlinkPathInfo = Deno.statSync(validSymlinkPath); - assert(symlinkPathInfo.isFile); - await Deno.remove(validSymlinkPath); +unitTest({ perms: { write: true } }, async function removeAllFail(): Promise< + void +> { + for (const method of REMOVE_METHODS) { + // NON-EXISTENT DIRECTORY/FILE let err; try { - Deno.statSync(validSymlinkPath); + // Non-existent + await Deno[method]("/baddir", { recursive: true }); } catch (e) { err = e; } - Deno.removeSync(filePath); assert(err instanceof Deno.errors.NotFound); } -); +}); -unitTest({ perms: { write: false } }, async function removePerm(): Promise< +unitTest({ perms: { write: false } }, async function removeAllPerm(): Promise< void > { - let err; - try { - await Deno.remove("/baddir"); - } catch (e) { - err = e; - } - assert(err instanceof Deno.errors.PermissionDenied); - assertEquals(err.name, "PermissionDenied"); -}); - -unitTest( - { perms: { write: true, read: true } }, - async function removeAllDirSuccess(): Promise { - // REMOVE EMPTY DIRECTORY - let path = Deno.makeTempDirSync() + "/dir/subdir"; - Deno.mkdirSync(path, { recursive: true }); - let pathInfo = Deno.statSync(path); - assert(pathInfo.isDirectory); // check exist first - await Deno.remove(path, { recursive: true }); // remove - // We then check again after remove + for (const method of REMOVE_METHODS) { let err; try { - Deno.statSync(path); + await Deno[method]("/baddir", { recursive: true }); } catch (e) { err = e; } - // Directory is gone - assert(err instanceof Deno.errors.NotFound); - - // REMOVE NON-EMPTY DIRECTORY - path = Deno.makeTempDirSync() + "/dir/subdir"; - const subPath = path + "/subsubdir"; - Deno.mkdirSync(path, { recursive: true }); - Deno.mkdirSync(subPath); - pathInfo = Deno.statSync(path); - assert(pathInfo.isDirectory); // check exist first - const subPathInfo = Deno.statSync(subPath); - assert(subPathInfo.isDirectory); // check exist first - await Deno.remove(path, { recursive: true }); // remove - // We then check parent directory again after remove - try { - Deno.statSync(path); - } catch (e) { - err = e; - } - // Directory is gone - assert(err instanceof Deno.errors.NotFound); + assert(err instanceof Deno.errors.PermissionDenied); + assertEquals(err.name, "PermissionDenied"); } -); +}); unitTest( - { perms: { write: true, read: true } }, - async function removeAllFileSuccess(): Promise { - // REMOVE FILE - const enc = new TextEncoder(); - const data = enc.encode("Hello"); - const filename = Deno.makeTempDirSync() + "/test.txt"; - Deno.writeFileSync(filename, data, { mode: 0o666 }); - const fileInfo = Deno.statSync(filename); - assert(fileInfo.isFile); // check exist first - await Deno.remove(filename, { recursive: true }); // remove - // We then check again after remove - let err; - try { - Deno.statSync(filename); - } catch (e) { - err = e; + { + ignore: Deno.build.os === "windows", + perms: { write: true, read: true }, + }, + async function removeUnixSocketSuccess(): Promise { + for (const method of REMOVE_METHODS) { + // MAKE TEMPORARY UNIX SOCKET + const path = Deno.makeTempDirSync() + "/test.sock"; + const listener = Deno.listen({ transport: "unix", path }); + listener.close(); + Deno.statSync(path); // check if unix socket exists + + await Deno[method](path); + let err; + try { + Deno.statSync(path); + } catch (e) { + err = e; + } + assert(err instanceof Deno.errors.NotFound); } - // File is gone - assert(err instanceof Deno.errors.NotFound); } ); -unitTest({ perms: { write: true } }, async function removeAllFail(): Promise< - void -> { - // NON-EXISTENT DIRECTORY/FILE - let err; - try { - // Non-existent - await Deno.remove("/baddir", { recursive: true }); - } catch (e) { - err = e; - } - assert(err instanceof Deno.errors.NotFound); -}); - -unitTest({ perms: { write: false } }, async function removeAllPerm(): Promise< - void -> { - let err; - try { - await Deno.remove("/baddir", { recursive: true }); - } catch (e) { - err = e; - } - assert(err instanceof Deno.errors.PermissionDenied); - assertEquals(err.name, "PermissionDenied"); -}); - if (Deno.build.os === "windows") { unitTest( { perms: { run: true, write: true, read: true } }, diff --git a/cli/tests/unit/request_test.ts b/cli/tests/unit/request_test.ts index 8a276c5e79284b..24f5b6d823b71c 100644 --- a/cli/tests/unit/request_test.ts +++ b/cli/tests/unit/request_test.ts @@ -10,22 +10,22 @@ unitTest(function fromInit(): void { }, }); - // @ts-ignore - assertEquals("ahoyhoy", req._bodySource); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + assertEquals("ahoyhoy", (req as any)._bodySource); assertEquals(req.url, "https://example.com"); assertEquals(req.headers.get("test-header"), "value"); }); unitTest(function fromRequest(): void { const r = new Request("https://example.com"); - // @ts-ignore - r._bodySource = "ahoyhoy"; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (r as any)._bodySource = "ahoyhoy"; r.headers.set("test-header", "value"); const req = new Request(r); - // @ts-ignore - assertEquals(req._bodySource, r._bodySource); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + assertEquals((req as any)._bodySource, (r as any)._bodySource); assertEquals(req.url, r.url); assertEquals(req.headers.get("test-header"), r.headers.get("test-header")); }); @@ -44,6 +44,6 @@ unitTest(async function cloneRequestBodyStream(): Promise { assertEquals(b1, b2); - // @ts-ignore - assert(r1._bodySource !== r2._bodySource); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + assert((r1 as any)._bodySource !== (r2 as any)._bodySource); }); diff --git a/cli/tests/unit/streams_internal_test.ts b/cli/tests/unit/streams_internal_test.ts index f324da194500e0..346ec27af50dc5 100644 --- a/cli/tests/unit/streams_internal_test.ts +++ b/cli/tests/unit/streams_internal_test.ts @@ -2,15 +2,12 @@ import { unitTest, assertThrows } from "./test_util.ts"; unitTest(function streamReadableHwmError() { - const invalidHwm = [NaN, Number("NaN"), {}, -1, "two"]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const invalidHwm: any[] = [NaN, Number("NaN"), {}, -1, "two"]; for (const highWaterMark of invalidHwm) { assertThrows( () => { - new ReadableStream( - undefined, - // @ts-ignore - { highWaterMark } - ); + new ReadableStream(undefined, { highWaterMark }); }, RangeError, "highWaterMark must be a positive number or Infinity. Received:" @@ -20,20 +17,20 @@ unitTest(function streamReadableHwmError() { assertThrows(() => { new ReadableStream( undefined, - // @ts-ignore - { highWaterMark: Symbol("hwk") } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + { highWaterMark: Symbol("hwk") as any } ); }, TypeError); }); unitTest(function streamWriteableHwmError() { - const invalidHwm = [NaN, Number("NaN"), {}, -1, "two"]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const invalidHwm: any[] = [NaN, Number("NaN"), {}, -1, "two"]; for (const highWaterMark of invalidHwm) { assertThrows( () => { new WritableStream( undefined, - // @ts-ignore new CountQueuingStrategy({ highWaterMark }) ); }, @@ -45,23 +42,19 @@ unitTest(function streamWriteableHwmError() { assertThrows(() => { new WritableStream( undefined, - // @ts-ignore - new CountQueuingStrategy({ highWaterMark: Symbol("hwmk") }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new CountQueuingStrategy({ highWaterMark: Symbol("hwmk") as any }) ); }, TypeError); }); unitTest(function streamTransformHwmError() { - const invalidHwm = [NaN, Number("NaN"), {}, -1, "two"]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const invalidHwm: any[] = [NaN, Number("NaN"), {}, -1, "two"]; for (const highWaterMark of invalidHwm) { assertThrows( () => { - new TransformStream( - undefined, - undefined, - // @ts-ignore - { highWaterMark } - ); + new TransformStream(undefined, undefined, { highWaterMark }); }, RangeError, "highWaterMark must be a positive number or Infinity. Received:" @@ -72,8 +65,8 @@ unitTest(function streamTransformHwmError() { new TransformStream( undefined, undefined, - // @ts-ignore - { highWaterMark: Symbol("hwmk") } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + { highWaterMark: Symbol("hwmk") as any } ); }, TypeError); }); diff --git a/cli/tests/unit/test_util.ts b/cli/tests/unit/test_util.ts index 1c5b6ff212d1aa..d5010fc630dc0c 100644 --- a/cli/tests/unit/test_util.ts +++ b/cli/tests/unit/test_util.ts @@ -7,8 +7,8 @@ export { assertEquals, assertMatch, assertNotEquals, - assertStrictEq, - assertStrContains, + assertStrictEquals, + assertStringContains, unreachable, fail, } from "../../../std/testing/asserts.ts"; diff --git a/cli/tests/unit/unit_test_runner.ts b/cli/tests/unit/unit_test_runner.ts index 715dda50010654..e3df358d7cf3cb 100755 --- a/cli/tests/unit/unit_test_runner.ts +++ b/cli/tests/unit/unit_test_runner.ts @@ -11,7 +11,7 @@ import { reportToConn, } from "./test_util.ts"; -// @ts-ignore +// @ts-expect-error const internalObj = Deno[Deno.internal]; // eslint-disable-next-line @typescript-eslint/no-explicit-any const reportToConsole = internalObj.reportToConsole as (message: any) => void; diff --git a/cli/tests/unit/url_search_params_test.ts b/cli/tests/unit/url_search_params_test.ts index 7b7dbab764059a..227deeda7a48a7 100644 --- a/cli/tests/unit/url_search_params_test.ts +++ b/cli/tests/unit/url_search_params_test.ts @@ -177,8 +177,8 @@ unitTest(function urlSearchParamsAppendArgumentsCheck(): void { const searchParams = new URLSearchParams(); let hasThrown = 0; try { - // @ts-ignore - searchParams[method](); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (searchParams as any)[method](); hasThrown = 1; } catch (err) { if (err instanceof TypeError) { @@ -194,8 +194,8 @@ unitTest(function urlSearchParamsAppendArgumentsCheck(): void { const searchParams = new URLSearchParams(); let hasThrown = 0; try { - // @ts-ignore - searchParams[method]("foo"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (searchParams as any)[method]("foo"); hasThrown = 1; } catch (err) { if (err instanceof TypeError) { @@ -235,8 +235,10 @@ unitTest(function urlSearchParamsCustomSymbolIterator(): void { unitTest( function urlSearchParamsCustomSymbolIteratorWithNonStringParams(): void { const params = {}; - // @ts-ignore - params[Symbol.iterator] = function* (): IterableIterator<[number, number]> { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (params as any)[Symbol.iterator] = function* (): IterableIterator< + [number, number] + > { yield [1, 2]; }; const params1 = new URLSearchParams((params as unknown) as string[][]); diff --git a/cli/tests/unstable_disabled.out b/cli/tests/unstable_disabled.out index 695ca122cec46a..e951c5d0abd192 100644 --- a/cli/tests/unstable_disabled.out +++ b/cli/tests/unstable_disabled.out @@ -1,5 +1,5 @@ [WILDCARD] -error: TS2339 [ERROR]: Property 'loadavg' does not exist on type 'typeof Deno'. +error: TS2339 [ERROR]: Property 'loadavg' does not exist on type 'typeof Deno'. 'Deno.loadavg' is an unstable API. Did you forget to run with the '--unstable' flag? console.log(Deno.loadavg); ~~~~~~~ at [WILDCARD]/cli/tests/unstable.ts:1:18 diff --git a/cli/tsc.rs b/cli/tsc.rs index 664721bc15135b..1b3208e7534965 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -5,10 +5,9 @@ use crate::diagnostics::DiagnosticItem; use crate::disk_cache::DiskCache; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; -use crate::fmt; -use crate::fs as deno_fs; use crate::global_state::GlobalState; use crate::import_map::ImportMap; +use crate::module_graph::ModuleGraphFile; use crate::module_graph::ModuleGraphLoader; use crate::msg; use crate::op_error::OpError; @@ -16,7 +15,6 @@ use crate::ops; use crate::permissions::Permissions; use crate::source_maps::SourceMapGetter; use crate::startup_data; -use crate::state::exit_unstable; use crate::state::State; use crate::version; use crate::web_worker::WebWorker; @@ -50,73 +48,69 @@ use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; use std::task::Poll; -use std::time::Instant; use url::Url; -// TODO(bartlomieju): make static -pub fn get_available_libs() -> Vec { - vec![ - "deno.ns".to_string(), - "deno.window".to_string(), - "deno.worker".to_string(), - "deno.shared_globals".to_string(), - "deno.unstable".to_string(), - "dom".to_string(), - "dom.iterable".to_string(), - "es5".to_string(), - "es6".to_string(), - "esnext".to_string(), - "es2020".to_string(), - "es2020.full".to_string(), - "es2019".to_string(), - "es2019.full".to_string(), - "es2018".to_string(), - "es2018.full".to_string(), - "es2017".to_string(), - "es2017.full".to_string(), - "es2016".to_string(), - "es2016.full".to_string(), - "es2015".to_string(), - "es2015.collection".to_string(), - "es2015.core".to_string(), - "es2015.generator".to_string(), - "es2015.iterable".to_string(), - "es2015.promise".to_string(), - "es2015.proxy".to_string(), - "es2015.reflect".to_string(), - "es2015.symbol".to_string(), - "es2015.symbol.wellknown".to_string(), - "es2016.array.include".to_string(), - "es2017.intl".to_string(), - "es2017.object".to_string(), - "es2017.sharedmemory".to_string(), - "es2017.string".to_string(), - "es2017.typedarrays".to_string(), - "es2018.asyncgenerator".to_string(), - "es2018.asynciterable".to_string(), - "es2018.intl".to_string(), - "es2018.promise".to_string(), - "es2018.regexp".to_string(), - "es2019.array".to_string(), - "es2019.object".to_string(), - "es2019.string".to_string(), - "es2019.symbol".to_string(), - "es2020.bigint".to_string(), - "es2020.promise".to_string(), - "es2020.string".to_string(), - "es2020.symbol.wellknown".to_string(), - "esnext.array".to_string(), - "esnext.asynciterable".to_string(), - "esnext.bigint".to_string(), - "esnext.intl".to_string(), - "esnext.promise".to_string(), - "esnext.string".to_string(), - "esnext.symbol".to_string(), - "scripthost".to_string(), - "webworker".to_string(), - "webworker.importscripts".to_string(), - ] -} +pub const AVAILABLE_LIBS: &[&str] = &[ + "deno.ns", + "deno.window", + "deno.worker", + "deno.shared_globals", + "deno.unstable", + "dom", + "dom.iterable", + "es5", + "es6", + "esnext", + "es2020", + "es2020.full", + "es2019", + "es2019.full", + "es2018", + "es2018.full", + "es2017", + "es2017.full", + "es2016", + "es2016.full", + "es2015", + "es2015.collection", + "es2015.core", + "es2015.generator", + "es2015.iterable", + "es2015.promise", + "es2015.proxy", + "es2015.reflect", + "es2015.symbol", + "es2015.symbol.wellknown", + "es2016.array.include", + "es2017.intl", + "es2017.object", + "es2017.sharedmemory", + "es2017.string", + "es2017.typedarrays", + "es2018.asyncgenerator", + "es2018.asynciterable", + "es2018.intl", + "es2018.promise", + "es2018.regexp", + "es2019.array", + "es2019.object", + "es2019.string", + "es2019.symbol", + "es2020.bigint", + "es2020.promise", + "es2020.string", + "es2020.symbol.wellknown", + "esnext.array", + "esnext.asynciterable", + "esnext.bigint", + "esnext.intl", + "esnext.promise", + "esnext.string", + "esnext.symbol", + "scripthost", + "webworker", + "webworker.importscripts", +]; #[derive(Debug, Clone)] pub struct CompiledModule { @@ -160,6 +154,7 @@ impl Future for CompilerWorker { } } +// TODO(bartlomieju): use JSONC parser from dprint instead of Regex lazy_static! { static ref CHECK_JS_RE: Regex = Regex::new(r#""checkJs"\s*?:\s*?true"#).unwrap(); @@ -175,9 +170,14 @@ fn create_compiler_worker( // like 'eval', 'repl' let entry_point = ModuleSpecifier::resolve_url_or_path("./__$deno$ts_compiler.ts").unwrap(); - let worker_state = - State::new(global_state.clone(), Some(permissions), entry_point, true) - .expect("Unable to create worker state"); + let worker_state = State::new( + global_state.clone(), + Some(permissions), + entry_point, + None, + true, + ) + .expect("Unable to create worker state"); // TODO(bartlomieju): this metric is never used anywhere // Count how many times we start the compiler worker. @@ -294,6 +294,28 @@ impl CompiledFileMetadata { } } +/// Information associated with compilation of a "module graph", +/// ie. entry point and all its dependencies. +/// It's used to perform cache invalidation if content of any +/// dependency changes. +#[derive(Deserialize, Serialize)] +pub struct GraphFileMetadata { + pub deps: Vec, + pub version_hash: String, +} + +impl GraphFileMetadata { + pub fn from_json_string( + metadata_string: String, + ) -> Result { + serde_json::from_str::(&metadata_string) + } + + pub fn to_json_string(&self) -> Result { + serde_json::to_string(self) + } +} + /// Emit a SHA256 hash based on source code, deno version and TS config. /// Used to check if a recompilation for source code is needed. pub fn source_code_version_hash( @@ -383,6 +405,7 @@ impl TsCompiler { }))) } + // TODO(bartlomieju): this method is no longer needed /// Mark given module URL as compiled to avoid multiple compilations of same /// module in single run. fn mark_compiled(&self, url: &Url) { @@ -390,11 +413,34 @@ impl TsCompiler { c.insert(url.clone()); } - /// Check if given module URL has already been compiled and can be fetched - /// directly from disk. - fn has_compiled(&self, url: &Url) -> bool { - let c = self.compiled.lock().unwrap(); - c.contains(url) + /// Check if there is compiled source in cache that is valid + /// and can be used again. + // TODO(bartlomieju): there should be check that cached file actually exists + fn has_compiled_source( + &self, + file_fetcher: &SourceFileFetcher, + url: &Url, + ) -> bool { + let specifier = ModuleSpecifier::from(url.clone()); + if let Some(source_file) = file_fetcher + .fetch_cached_source_file(&specifier, Permissions::allow_all()) + { + if let Some(metadata) = self.get_metadata(&url) { + // 2. compare version hashes + // TODO: it would probably be good idea to make it method implemented on SourceFile + let version_hash_to_validate = source_code_version_hash( + &source_file.source_code, + version::DENO, + &self.config.hash, + ); + + if metadata.version_hash == version_hash_to_validate { + return true; + } + } + } + + false } /// Asynchronously compile module and all it's dependencies. @@ -406,64 +452,43 @@ impl TsCompiler { /// /// If compilation is required then new V8 worker is spawned with fresh TS /// compiler. - pub async fn compile( + pub async fn compile_module_graph( &self, global_state: GlobalState, source_file: &SourceFile, target: TargetLib, permissions: Permissions, - is_dyn_import: bool, - ) -> Result { - if self.has_compiled(&source_file.url) { - return self.get_compiled_module(&source_file.url); - } + module_graph: HashMap, + ) -> Result<(), ErrBox> { + let mut has_cached_version = false; if self.use_disk_cache { - // Try to load cached version: - // 1. check if there's 'meta' file - if let Some(metadata) = self.get_metadata(&source_file.url) { - // 2. compare version hashes - // TODO: it would probably be good idea to make it method implemented on SourceFile - let version_hash_to_validate = source_code_version_hash( - &source_file.source_code, - version::DENO, + if let Some(metadata) = self.get_graph_metadata(&source_file.url) { + has_cached_version = true; + + let version_hash = crate::checksum::gen(vec![ + version::DENO.as_bytes(), &self.config.hash, - ); + ]); - if metadata.version_hash == version_hash_to_validate { - debug!("load_cache metadata version hash match"); - if let Ok(compiled_module) = - self.get_compiled_module(&source_file.url) - { - self.mark_compiled(&source_file.url); - return Ok(compiled_module); - } + has_cached_version &= metadata.version_hash == version_hash; + has_cached_version &= self + .has_compiled_source(&global_state.file_fetcher, &source_file.url); + + for dep in metadata.deps { + let url = Url::parse(&dep).expect("Dep is not a valid url"); + has_cached_version &= + self.has_compiled_source(&global_state.file_fetcher, &url); } } } - let source_file_ = source_file.clone(); + + if has_cached_version { + return Ok(()); + } + let module_url = source_file.url.clone(); - let module_specifier = ModuleSpecifier::from(source_file.url.clone()); - let import_map: Option = - match global_state.flags.import_map_path.as_ref() { - None => None, - Some(file_path) => { - if !global_state.flags.unstable { - exit_unstable("--importmap") - } - Some(ImportMap::load(file_path)?) - } - }; - let mut module_graph_loader = ModuleGraphLoader::new( - global_state.file_fetcher.clone(), - import_map, - permissions.clone(), - is_dyn_import, - true, - ); - module_graph_loader.add_to_graph(&module_specifier).await?; - let module_graph = module_graph_loader.get_graph(); let module_graph_json = serde_json::to_value(module_graph).expect("Failed to serialize data"); let target = match target { @@ -500,23 +525,17 @@ impl TsCompiler { let req_msg = j.to_string().into_boxed_str().into_boxed_bytes(); - let ts_compiler = self.clone(); - + // TODO(bartlomieju): lift this call up - TSC shouldn't print anything info!( "{} {}", colors::green("Compile".to_string()), module_url.to_string() ); - let start = Instant::now(); - let msg = execute_in_same_thread(global_state.clone(), permissions, req_msg) .await?; - let end = Instant::now(); - debug!("time spent in compiler thread {:#?}", end - start); - let json_str = std::str::from_utf8(&msg).unwrap(); let compile_response: CompileResponse = serde_json::from_str(json_str)?; @@ -525,8 +544,69 @@ impl TsCompiler { return Err(ErrBox::from(compile_response.diagnostics)); } + self.set_graph_metadata( + source_file.url.clone(), + &compile_response.emit_map, + )?; self.cache_emitted_files(compile_response.emit_map)?; - ts_compiler.get_compiled_module(&source_file_.url) + Ok(()) + } + + fn get_graph_metadata(&self, url: &Url) -> Option { + // Try to load cached version: + // 1. check if there's 'meta' file + let cache_key = self + .disk_cache + .get_cache_filename_with_extension(url, "graph"); + if let Ok(metadata_bytes) = self.disk_cache.get(&cache_key) { + if let Ok(metadata) = std::str::from_utf8(&metadata_bytes) { + if let Ok(read_metadata) = + GraphFileMetadata::from_json_string(metadata.to_string()) + { + return Some(read_metadata); + } + } + } + + None + } + + fn set_graph_metadata( + &self, + url: Url, + emit_map: &HashMap, + ) -> std::io::Result<()> { + let version_hash = + crate::checksum::gen(vec![version::DENO.as_bytes(), &self.config.hash]); + let mut deps = vec![]; + + for (_emitted_name, source) in emit_map.iter() { + let specifier = ModuleSpecifier::resolve_url(&source.filename) + .expect("Should be a valid module specifier"); + + let source_file = self + .file_fetcher + .fetch_cached_source_file(&specifier, Permissions::allow_all()) + .expect("Source file not found"); + + // NOTE: JavaScript files are only cached to disk if `checkJs` + // option in on + if source_file.media_type == msg::MediaType::JavaScript + && !self.compile_js + { + continue; + } + + deps.push(specifier.to_string()); + } + + let graph_metadata = GraphFileMetadata { deps, version_hash }; + let meta_key = self + .disk_cache + .get_cache_filename_with_extension(&url, "graph"); + self + .disk_cache + .set(&meta_key, graph_metadata.to_json_string()?.as_bytes()) } /// Get associated `CompiledFileMetadata` for given module if it exists. @@ -557,10 +637,23 @@ impl TsCompiler { let specifier = ModuleSpecifier::resolve_url(&source.filename) .expect("Should be a valid module specifier"); + let source_file = self + .file_fetcher + .fetch_cached_source_file(&specifier, Permissions::allow_all()) + .expect("Source file not found"); + + // NOTE: JavaScript files are only cached to disk if `checkJs` + // option in on + if source_file.media_type == msg::MediaType::JavaScript + && !self.compile_js + { + continue; + } + if emitted_name.ends_with(".map") { self.cache_source_map(&specifier, &source.contents)?; } else if emitted_name.ends_with(".js") { - self.cache_compiled_file(&specifier, &source.contents)?; + self.cache_compiled_file(&specifier, source_file, &source.contents)?; } else { panic!("Trying to cache unknown file type {}", emitted_name); } @@ -618,20 +711,9 @@ impl TsCompiler { fn cache_compiled_file( &self, module_specifier: &ModuleSpecifier, + source_file: SourceFile, contents: &str, ) -> std::io::Result<()> { - let source_file = self - .file_fetcher - .fetch_cached_source_file(&module_specifier, Permissions::allow_all()) - .expect("Source file not found"); - - // NOTE: JavaScript files are only cached to disk if `checkJs` - // option in on - if source_file.media_type == msg::MediaType::JavaScript && !self.compile_js - { - return Ok(()); - } - // By default TSC output source map url that is relative; we need // to substitute it manually to correct file URL in DENO_DIR. let mut content_lines = contents @@ -664,10 +746,6 @@ impl TsCompiler { .get_cache_filename_with_extension(module_specifier.as_url(), "js"); self.disk_cache.set(&js_key, contents.as_bytes())?; self.mark_compiled(module_specifier.as_url()); - let source_file = self - .file_fetcher - .fetch_cached_source_file(&module_specifier, Permissions::allow_all()) - .expect("Source file not found"); let version_hash = source_code_version_hash( &source_file.source_code, @@ -720,18 +798,6 @@ impl TsCompiler { module_specifier: &ModuleSpecifier, contents: &str, ) -> std::io::Result<()> { - let source_file = self - .file_fetcher - .fetch_cached_source_file(&module_specifier, Permissions::allow_all()) - .expect("Source file not found"); - - // NOTE: JavaScript files are only cached to disk if `checkJs` - // option in on - if source_file.media_type == msg::MediaType::JavaScript && !self.compile_js - { - return Ok(()); - } - let js_key = self .disk_cache .get_cache_filename_with_extension(module_specifier.as_url(), "js"); @@ -767,12 +833,12 @@ impl SourceMapGetter for TsCompiler { self .try_resolve_and_get_source_file(script_name) .and_then(|out| { - str::from_utf8(&out.source_code).ok().and_then(|v| { + str::from_utf8(&out.source_code).ok().map(|v| { // Do NOT use .lines(): it skips the terminating empty line. // (due to internally using .split_terminator() instead of .split()) let lines: Vec<&str> = v.split('\n').collect(); assert!(lines.len() > line); - Some(lines[line].to_string()) + lines[line].to_string() }) }) } @@ -854,14 +920,12 @@ pub async fn bundle( compiler_config: CompilerConfig, module_specifier: ModuleSpecifier, maybe_import_map: Option, - out_file: Option, unstable: bool, -) -> Result<(), ErrBox> { +) -> Result { debug!( "Invoking the compiler to bundle. module_name: {}", module_specifier.to_string() ); - eprintln!("Bundling {}", module_specifier.to_string()); let permissions = Permissions::allow_all(); let mut module_graph_loader = ModuleGraphLoader::new( @@ -871,7 +935,9 @@ pub async fn bundle( false, true, ); - module_graph_loader.add_to_graph(&module_specifier).await?; + module_graph_loader + .add_to_graph(&module_specifier, None) + .await?; let module_graph = module_graph_loader.get_graph(); let module_graph_json = serde_json::to_value(module_graph).expect("Failed to serialize data"); @@ -921,26 +987,7 @@ pub async fn bundle( assert!(bundle_response.bundle_output.is_some()); let output = bundle_response.bundle_output.unwrap(); - - // TODO(bartlomieju): the rest of this function should be handled - // in `main.rs` - it has nothing to do with TypeScript... - let output_string = fmt::format_text(&output)?; - - if let Some(out_file_) = out_file.as_ref() { - eprintln!("Emitting bundle to {:?}", out_file_); - - let output_bytes = output_string.as_bytes(); - let output_len = output_bytes.len(); - - deno_fs::write_file(out_file_, output_bytes, 0o666)?; - // TODO(bartlomieju): do we really need to show this info? (it doesn't respect --quiet flag) - // TODO(bartlomieju): add "humanFileSize" method - eprintln!("{} bytes emitted.", output_len); - } else { - println!("{}", output_string); - } - - Ok(()) + Ok(output) } /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs. @@ -968,7 +1015,9 @@ pub async fn runtime_compile( let module_specifier = ModuleSpecifier::resolve_import(root_name, "")?; root_names.push(module_specifier.to_string()); - module_graph_loader.add_to_graph(&module_specifier).await?; + module_graph_loader + .add_to_graph(&module_specifier, None) + .await?; } // download all additional files from TSconfig and add them to root_names @@ -983,7 +1032,9 @@ pub async fn runtime_compile( .expect("type is not a string") .to_string(); let type_specifier = ModuleSpecifier::resolve_url_or_path(&type_str)?; - module_graph_loader.add_to_graph(&type_specifier).await?; + module_graph_loader + .add_to_graph(&type_specifier, None) + .await?; root_names.push(type_specifier.to_string()) } } @@ -1078,18 +1129,36 @@ mod tests { }; let mock_state = GlobalState::mock(vec![String::from("deno"), String::from("hello.ts")]); + + let mut module_graph_loader = ModuleGraphLoader::new( + mock_state.file_fetcher.clone(), + None, + Permissions::allow_all(), + false, + false, + ); + module_graph_loader + .add_to_graph(&specifier, None) + .await + .expect("Failed to create graph"); + let module_graph = module_graph_loader.get_graph(); + let result = mock_state .ts_compiler - .compile( + .compile_module_graph( mock_state.clone(), &out, TargetLib::Main, Permissions::allow_all(), - false, + module_graph, ) .await; assert!(result.is_ok()); - let source_code = result.unwrap().code; + let compiled_file = mock_state + .ts_compiler + .get_compiled_module(&out.url) + .unwrap(); + let source_code = compiled_file.code; assert!(source_code .as_bytes() .starts_with(b"\"use strict\";\nconsole.log(\"Hello World\");")); @@ -1143,7 +1212,6 @@ mod tests { CompilerConfig::load(None).unwrap(), module_name, None, - None, false, ) .await; diff --git a/cli/upgrade.rs b/cli/upgrade.rs index b4f207643b0b9d..dcf166a71c4b4a 100644 --- a/cli/upgrade.rs +++ b/cli/upgrade.rs @@ -172,28 +172,40 @@ fn unpack(archive_data: Vec) -> Result { cmd.stdin.as_mut().unwrap().write_all(&archive_data)?; cmd.wait()? } + "zip" if cfg!(windows) => { + let archive_path = temp_dir.join("deno.zip"); + fs::write(&archive_path, &archive_data)?; + Command::new("powershell.exe") + .arg("-NoLogo") + .arg("-NoProfile") + .arg("-NonInteractive") + .arg("-Command") + .arg( + "& { + param($Path, $DestinationPath) + trap { $host.ui.WriteErrorLine($_.Exception); exit 1 } + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory( + $Path, + $DestinationPath + ); + }", + ) + .arg("-Path") + .arg(&archive_path) + .arg("-DestinationPath") + .arg(&temp_dir) + .spawn()? + .wait()? + } "zip" => { - if cfg!(windows) { - let archive_path = temp_dir.join("deno.zip"); - fs::write(&archive_path, &archive_data)?; - Command::new("powershell.exe") - .arg("-Command") - .arg("Expand-Archive") - .arg("-Path") - .arg(&archive_path) - .arg("-DestinationPath") - .arg(&temp_dir) - .spawn()? - .wait()? - } else { - let archive_path = temp_dir.join("deno.zip"); - fs::write(&archive_path, &archive_data)?; - Command::new("unzip") - .current_dir(&temp_dir) - .arg(archive_path) - .spawn()? - .wait()? - } + let archive_path = temp_dir.join("deno.zip"); + fs::write(&archive_path, &archive_data)?; + Command::new("unzip") + .current_dir(&temp_dir) + .arg(archive_path) + .spawn()? + .wait()? } ext => panic!("Unsupported archive type: '{}'", ext), }; diff --git a/cli/web_worker.rs b/cli/web_worker.rs index 46f03da363b226..e060a157d62f31 100644 --- a/cli/web_worker.rs +++ b/cli/web_worker.rs @@ -91,12 +91,7 @@ impl WebWorker { let mut worker = Worker::new(name, startup_data, state_); let terminated = Arc::new(AtomicBool::new(false)); - let isolate_handle = worker - .isolate - .v8_isolate - .as_mut() - .unwrap() - .thread_safe_handle(); + let isolate_handle = worker.isolate.thread_safe_handle(); let (terminate_tx, terminate_rx) = mpsc::channel::<()>(1); let handle = WebWorkerHandle { diff --git a/cli/worker.rs b/cli/worker.rs index b66c1cb06b1548..70cadcc23b170f 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -1,9 +1,13 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. use crate::fmt_errors::JSError; +use crate::global_state::GlobalState; use crate::inspector::DenoInspector; use crate::ops; +use crate::ops::io::get_stdio; +use crate::startup_data; use crate::state::State; use deno_core::Buf; +use deno_core::CoreIsolate; use deno_core::ErrBox; use deno_core::ModuleId; use deno_core::ModuleSpecifier; @@ -12,7 +16,6 @@ use futures::channel::mpsc; use futures::future::FutureExt; use futures::stream::StreamExt; use futures::task::AtomicWaker; -use std::cell::RefMut; use std::env; use std::future::Future; use std::ops::Deref; @@ -87,7 +90,8 @@ fn create_channels() -> (WorkerChannelsInternal, WorkerHandle) { /// - `WebWorker` pub struct Worker { pub name: String, - pub isolate: Box, + pub isolate: deno_core::EsIsolate, + pub inspector: Option>, pub state: State, pub waker: AtomicWaker, pub(crate) internal_channels: WorkerChannelsInternal, @@ -99,18 +103,32 @@ impl Worker { let loader = Rc::new(state.clone()); let mut isolate = deno_core::EsIsolate::new(loader, startup_data, false); - state.maybe_init_inspector(&mut isolate); + { + let global_state = state.borrow().global_state.clone(); + let core_state_rc = CoreIsolate::state(&isolate); + let mut core_state = core_state_rc.borrow_mut(); + core_state.set_js_error_create_fn(move |core_js_error| { + JSError::create(core_js_error, &global_state.ts_compiler) + }); + } - let global_state = state.borrow().global_state.clone(); - isolate.set_js_error_create_fn(move |core_js_error| { - JSError::create(core_js_error, &global_state.ts_compiler) - }); + let inspector = { + let state = state.borrow(); + let global_state = &state.global_state; + global_state + .flags + .inspect + .or(global_state.flags.inspect_brk) + .filter(|_| !state.is_internal) + .map(|inspector_host| DenoInspector::new(&mut isolate, inspector_host)) + }; let (internal_channels, external_channels) = create_channels(); Self { name, isolate, + inspector, state, waker: AtomicWaker::new(), internal_channels, @@ -173,16 +191,14 @@ impl Worker { self.external_channels.clone() } - #[inline(always)] - fn inspector(&self) -> RefMut>> { - let state = self.state.borrow_mut(); - RefMut::map(state, |s| &mut s.inspector) - } - - fn wait_for_inspector_session(&self) { - if self.state.should_inspector_break_on_first_statement() { + fn wait_for_inspector_session(&mut self) { + let should_break_on_first_statement = self.inspector.is_some() && { + let state = self.state.borrow(); + state.is_main && state.global_state.flags.inspect_brk.is_some() + }; + if should_break_on_first_statement { self - .inspector() + .inspector .as_mut() .unwrap() .wait_for_session_and_break_on_next_statement() @@ -194,7 +210,7 @@ impl Drop for Worker { fn drop(&mut self) { // The Isolate object must outlive the Inspector object, but this is // currently not enforced by the type system. - self.inspector().take(); + self.inspector.take(); } } @@ -205,7 +221,7 @@ impl Future for Worker { let inner = self.get_mut(); // We always poll the inspector if it exists. - let _ = inner.inspector().as_mut().map(|i| i.poll_unpin(cx)); + let _ = inner.inspector.as_mut().map(|i| i.poll_unpin(cx)); inner.waker.register(cx.waker()); inner.isolate.poll_unpin(cx) } @@ -233,7 +249,8 @@ impl DerefMut for Worker { pub struct MainWorker(Worker); impl MainWorker { - pub fn new(name: String, startup_data: StartupData, state: State) -> Self { + // TODO(ry) combine MainWorker::new and MainWorker::create. + fn new(name: String, startup_data: StartupData, state: State) -> Self { let state_ = state.clone(); let mut worker = Worker::new(name, startup_data, state_); { @@ -261,6 +278,35 @@ impl MainWorker { } Self(worker) } + + pub fn create( + global_state: GlobalState, + main_module: ModuleSpecifier, + ) -> Result { + let state = State::new( + global_state.clone(), + None, + main_module, + global_state.maybe_import_map.clone(), + false, + )?; + let mut worker = MainWorker::new( + "main".to_string(), + startup_data::deno_isolate_init(), + state, + ); + { + let (stdin, stdout, stderr) = get_stdio(); + let state_rc = CoreIsolate::state(&worker.isolate); + let state = state_rc.borrow(); + let mut t = state.resource_table.borrow_mut(); + t.add("stdin", Box::new(stdin)); + t.add("stdout", Box::new(stdout)); + t.add("stderr", Box::new(stderr)); + } + worker.execute("bootstrap.mainRuntime()")?; + Ok(worker) + } } impl Deref for MainWorker { @@ -296,7 +342,8 @@ mod tests { ModuleSpecifier::resolve_url_or_path(&p.to_string_lossy()).unwrap(); let global_state = GlobalState::new(flags::Flags::default()).unwrap(); let state = - State::new(global_state, None, module_specifier.clone(), false).unwrap(); + State::new(global_state, None, module_specifier.clone(), None, false) + .unwrap(); let state_ = state.clone(); tokio_util::run_basic(async move { let mut worker = @@ -325,7 +372,8 @@ mod tests { ModuleSpecifier::resolve_url_or_path(&p.to_string_lossy()).unwrap(); let global_state = GlobalState::new(flags::Flags::default()).unwrap(); let state = - State::new(global_state, None, module_specifier.clone(), false).unwrap(); + State::new(global_state, None, module_specifier.clone(), None, false) + .unwrap(); let state_ = state.clone(); tokio_util::run_basic(async move { let mut worker = @@ -340,7 +388,6 @@ mod tests { }); let state = state_.borrow(); - assert_eq!(state.metrics.resolve_count, 1); // Check that we didn't start the compiler. assert_eq!(state.global_state.compiler_starts.load(Ordering::SeqCst), 0); } @@ -362,9 +409,14 @@ mod tests { ..flags::Flags::default() }; let global_state = GlobalState::new(flags).unwrap(); - let state = - State::new(global_state.clone(), None, module_specifier.clone(), false) - .unwrap(); + let state = State::new( + global_state.clone(), + None, + module_specifier.clone(), + None, + false, + ) + .unwrap(); let mut worker = MainWorker::new( "TEST".to_string(), startup_data::deno_isolate_init(), diff --git a/core/Cargo.toml b/core/Cargo.toml index ebd10838b3b95c..93da1dc90a4f62 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_core" -version = "0.45.2" +version = "0.47.1" edition = "2018" description = "A secure JavaScript/TypeScript runtime built with V8, Rust, and Tokio" authors = ["the Deno authors"] @@ -15,12 +15,12 @@ path = "lib.rs" [dependencies] downcast-rs = "1.1.1" -futures = { version = "0.3.4", features = ["thread-pool", "compat"] } +futures = { version = "0.3.5", features = ["thread-pool", "compat"] } lazy_static = "1.4.0" -libc = "0.2.69" +libc = "0.2.71" log = "0.4.8" -rusty_v8 = "0.4.2" -serde_json = "1.0.52" +rusty_v8 = "0.5.0" +serde_json = "1.0.53" url = "2.1.1" [[example]] @@ -30,4 +30,4 @@ path = "examples/http_bench.rs" # These dependendencies are only used for deno_core_http_bench. [dev-dependencies] derive_deref = "1.1.0" -tokio = { version = "0.2.20", features = ["rt-core", "tcp"] } +tokio = { version = "0.2.21", features = ["rt-core", "tcp"] } diff --git a/core/any_error.rs b/core/any_error.rs deleted file mode 100644 index 34709e77fc38d1..00000000000000 --- a/core/any_error.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -use std::any::Any; -use std::any::TypeId; -use std::convert::From; -use std::error::Error; -use std::fmt; -use std::ops::Deref; - -// The Send and Sync traits are required because deno is multithreaded and we -// need to be able to handle errors across threads. -pub trait AnyError: Any + Error + Send + Sync + 'static {} -impl AnyError for T where T: Any + Error + Send + Sync + Sized + 'static {} - -#[derive(Debug)] -pub struct ErrBox(Box); - -impl dyn AnyError { - pub fn downcast_ref(&self) -> Option<&T> { - if Any::type_id(self) == TypeId::of::() { - let target = self as *const Self as *const T; - let target = unsafe { &*target }; - Some(target) - } else { - None - } - } -} - -impl ErrBox { - pub fn downcast(self) -> Result { - if Any::type_id(&*self.0) == TypeId::of::() { - let target = Box::into_raw(self.0) as *mut T; - let target = unsafe { Box::from_raw(target) }; - Ok(*target) - } else { - Err(self) - } - } -} - -impl AsRef for ErrBox { - fn as_ref(&self) -> &dyn AnyError { - self.0.as_ref() - } -} - -impl Deref for ErrBox { - type Target = Box; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl From for ErrBox { - fn from(error: T) -> Self { - Self(Box::new(error)) - } -} - -impl From> for ErrBox { - fn from(boxed: Box) -> Self { - Self(boxed) - } -} - -impl fmt::Display for ErrBox { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} diff --git a/core/bindings.rs b/core/bindings.rs index 16e78e368f0e10..9b66bfc481430f 100644 --- a/core/bindings.rs +++ b/core/bindings.rs @@ -1,9 +1,9 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -use crate::es_isolate::EsIsolate; -use crate::isolate::CoreIsolate; -use crate::isolate::ZeroCopyBuf; -use crate::js_errors::JSError; +use crate::CoreIsolate; +use crate::EsIsolate; +use crate::JSError; +use crate::ZeroCopyBuf; use rusty_v8 as v8; use v8::MapFnTo; @@ -239,7 +239,8 @@ pub fn boxed_slice_to_uint8array<'sc>( let backing_store = v8::ArrayBuffer::new_backing_store_from_boxed_slice(buf); let backing_store_shared = backing_store.make_shared(); let ab = v8::ArrayBuffer::with_backing_store(scope, &backing_store_shared); - v8::Uint8Array::new(ab, 0, buf_len).expect("Failed to create UintArray8") + v8::Uint8Array::new(scope, ab, 0, buf_len) + .expect("Failed to create UintArray8") } pub extern "C" fn host_import_module_dynamically_callback( @@ -250,9 +251,6 @@ pub extern "C" fn host_import_module_dynamically_callback( let mut cbs = v8::CallbackScope::new_escapable(context); let mut hs = v8::EscapableHandleScope::new(cbs.enter()); let scope = hs.enter(); - let isolate = scope.isolate(); - let core_isolate: &mut EsIsolate = - unsafe { &mut *(isolate.get_data(1) as *mut EsIsolate) }; // NOTE(bartlomieju): will crash for non-UTF-8 specifier let specifier_str = specifier @@ -277,11 +275,11 @@ pub extern "C" fn host_import_module_dynamically_callback( let mut resolver_handle = v8::Global::new(); resolver_handle.set(scope, resolver); - core_isolate.dyn_import_cb( - resolver_handle, - &specifier_str, - &referrer_name_str, - ); + { + let state_rc = EsIsolate::state(scope.isolate()); + let mut state = state_rc.borrow_mut(); + state.dyn_import_cb(resolver_handle, &specifier_str, &referrer_name_str); + } &mut *scope.escape(promise) } @@ -294,14 +292,13 @@ pub extern "C" fn host_initialize_import_meta_object_callback( let mut cbs = v8::CallbackScope::new(context); let mut hs = v8::HandleScope::new(cbs.enter()); let scope = hs.enter(); - let isolate = scope.isolate(); - let core_isolate: &mut EsIsolate = - unsafe { &mut *(isolate.get_data(1) as *mut EsIsolate) }; + let state_rc = EsIsolate::state(scope.isolate()); + let state = state_rc.borrow(); let id = module.get_identity_hash(); assert_ne!(id, 0); - let info = core_isolate.modules.get_info(id).expect("Module not found"); + let info = state.modules.get_info(id).expect("Module not found"); meta.create_data_property( context, @@ -320,10 +317,10 @@ pub extern "C" fn promise_reject_callback(message: v8::PromiseRejectMessage) { let mut hs = v8::HandleScope::new(cbs.enter()); let scope = hs.enter(); - let core_isolate: &mut CoreIsolate = - unsafe { &mut *(scope.isolate().get_data(0) as *mut CoreIsolate) }; + let state_rc = CoreIsolate::state(scope.isolate()); + let mut state = state_rc.borrow_mut(); - let context = core_isolate.global_context.get(scope).unwrap(); + let context = state.global_context.get(scope).unwrap(); let mut cs = v8::ContextScope::new(scope, context); let scope = cs.enter(); @@ -335,13 +332,13 @@ pub extern "C" fn promise_reject_callback(message: v8::PromiseRejectMessage) { let error = message.get_value(); let mut error_global = v8::Global::::new(); error_global.set(scope, error); - core_isolate + state .pending_promise_exceptions .insert(promise_id, error_global); } v8::PromiseRejectEvent::PromiseHandlerAddedAfterReject => { if let Some(mut handle) = - core_isolate.pending_promise_exceptions.remove(&promise_id) + state.pending_promise_exceptions.remove(&promise_id) { handle.reset(scope); } @@ -415,17 +412,17 @@ fn recv( args: v8::FunctionCallbackArguments, _rv: v8::ReturnValue, ) { - let core_isolate: &mut CoreIsolate = - unsafe { &mut *(scope.isolate().get_data(0) as *mut CoreIsolate) }; + let state_rc = CoreIsolate::state(scope.isolate()); + let mut state = state_rc.borrow_mut(); - if !core_isolate.js_recv_cb.is_empty() { + if !state.js_recv_cb.is_empty() { let msg = v8::String::new(scope, "Deno.core.recv already called.").unwrap(); scope.isolate().throw_exception(msg.into()); return; } let recv_fn = v8::Local::::try_from(args.get(0)).unwrap(); - core_isolate.js_recv_cb.set(scope, recv_fn); + state.js_recv_cb.set(scope, recv_fn); } fn send( @@ -433,10 +430,6 @@ fn send( args: v8::FunctionCallbackArguments, mut rv: v8::ReturnValue, ) { - let core_isolate: &mut CoreIsolate = - unsafe { &mut *(scope.isolate().get_data(0) as *mut CoreIsolate) }; - assert!(!core_isolate.global_context.is_empty()); - let op_id = match v8::Local::::try_from(args.get(0)) { Ok(op_id) => op_id.value() as u32, Err(err) => { @@ -450,7 +443,7 @@ fn send( let control_backing_store: v8::SharedRef; let control = match v8::Local::::try_from(args.get(1)) { Ok(view) => unsafe { - control_backing_store = view.buffer().unwrap().get_backing_store(); + control_backing_store = view.buffer(scope).unwrap().get_backing_store(); get_backing_store_slice( &control_backing_store, view.byte_offset(), @@ -460,14 +453,50 @@ fn send( Err(_) => &[], }; - let zero_copy: Option = - v8::Local::::try_from(args.get(2)) - .map(ZeroCopyBuf::new) - .ok(); + let state_rc = CoreIsolate::state(scope.isolate()); + let mut state = state_rc.borrow_mut(); + assert!(!state.global_context.is_empty()); + + let mut buf_iter = (2..args.length()).map(|idx| { + v8::Local::::try_from(args.get(idx)) + .map(|view| ZeroCopyBuf::new(scope, view)) + .map_err(|err| { + let msg = format!("Invalid argument at position {}: {}", idx, err); + let msg = v8::String::new(scope, &msg).unwrap(); + v8::Exception::type_error(scope, msg) + }) + }); + + let mut buf_one: ZeroCopyBuf; + let mut buf_vec: Vec; + + // Collect all ArrayBufferView's + let buf_iter_result = match buf_iter.len() { + 0 => Ok(&mut [][..]), + 1 => match buf_iter.next().unwrap() { + Ok(buf) => { + buf_one = buf; + Ok(std::slice::from_mut(&mut buf_one)) + } + Err(err) => Err(err), + }, + _ => match buf_iter.collect::, _>>() { + Ok(v) => { + buf_vec = v; + Ok(&mut buf_vec[..]) + } + Err(err) => Err(err), + }, + }; // If response is empty then it's either async op or exception was thrown - let maybe_response = - core_isolate.dispatch_op(scope, op_id, control, zero_copy); + let maybe_response = match buf_iter_result { + Ok(bufs) => state.dispatch_op(scope, op_id, control, bufs), + Err(exc) => { + scope.isolate().throw_exception(exc); + return; + } + }; if let Some(response) = maybe_response { // Synchronous response. @@ -476,7 +505,7 @@ fn send( if !buf.is_empty() { let ui8 = boxed_slice_to_uint8array(scope, buf); - rv.set(ui8.into()) + rv.set(ui8.into()); } } } @@ -486,10 +515,10 @@ fn set_macrotask_callback( args: v8::FunctionCallbackArguments, _rv: v8::ReturnValue, ) { - let core_isolate: &mut CoreIsolate = - unsafe { &mut *(scope.isolate().get_data(0) as *mut CoreIsolate) }; + let state_rc = CoreIsolate::state(scope.isolate()); + let mut state = state_rc.borrow_mut(); - if !core_isolate.js_macrotask_cb.is_empty() { + if !state.js_macrotask_cb.is_empty() { let msg = v8::String::new(scope, "Deno.core.setMacrotaskCallback already called.") .unwrap(); @@ -499,7 +528,7 @@ fn set_macrotask_callback( let macrotask_cb_fn = v8::Local::::try_from(args.get(0)).unwrap(); - core_isolate.js_macrotask_cb.set(scope, macrotask_cb_fn); + state.js_macrotask_cb.set(scope, macrotask_cb_fn); } fn eval_context( @@ -507,10 +536,12 @@ fn eval_context( args: v8::FunctionCallbackArguments, mut rv: v8::ReturnValue, ) { - let core_isolate: &mut CoreIsolate = - unsafe { &mut *(scope.isolate().get_data(0) as *mut CoreIsolate) }; - assert!(!core_isolate.global_context.is_empty()); - let context = core_isolate.global_context.get(scope).unwrap(); + let state_rc = CoreIsolate::state(scope.isolate()); + let context = { + let state = state_rc.borrow(); + assert!(!state.global_context.is_empty()); + state.global_context.get(scope).unwrap() + }; let source = match v8::Local::::try_from(args.get(0)) { Ok(s) => s, @@ -545,7 +576,7 @@ fn eval_context( if maybe_script.is_none() { assert!(tc.has_caught()); - let exception = tc.exception().unwrap(); + let exception = tc.exception(scope).unwrap(); output.set( context, @@ -586,7 +617,7 @@ fn eval_context( if result.is_none() { assert!(tc.has_caught()); - let exception = tc.exception().unwrap(); + let exception = tc.exception(scope).unwrap(); output.set( context, @@ -643,10 +674,10 @@ fn format_error( args: v8::FunctionCallbackArguments, mut rv: v8::ReturnValue, ) { - let core_isolate: &mut CoreIsolate = - unsafe { &mut *(scope.isolate().get_data(0) as *mut CoreIsolate) }; let e = JSError::from_v8_exception(scope, args.get(0)); - let e = (core_isolate.js_error_create_fn)(e); + let state_rc = CoreIsolate::state(scope.isolate()); + let state = state_rc.borrow(); + let e = (state.js_error_create_fn)(e); let e = e.to_string(); let e = v8::String::new(scope, &e).unwrap(); rv.set(e.into()) @@ -671,14 +702,15 @@ fn encode( let buf = if text_bytes.is_empty() { let ab = v8::ArrayBuffer::new(scope, 0); - v8::Uint8Array::new(ab, 0, 0).expect("Failed to create UintArray8") + v8::Uint8Array::new(scope, ab, 0, 0).expect("Failed to create UintArray8") } else { let buf_len = text_bytes.len(); let backing_store = v8::ArrayBuffer::new_backing_store_from_boxed_slice(text_bytes); let backing_store_shared = backing_store.make_shared(); let ab = v8::ArrayBuffer::with_backing_store(scope, &backing_store_shared); - v8::Uint8Array::new(ab, 0, buf_len).expect("Failed to create UintArray8") + v8::Uint8Array::new(scope, ab, 0, buf_len) + .expect("Failed to create UintArray8") }; rv.set(buf.into()) @@ -699,7 +731,7 @@ fn decode( } }; - let backing_store = view.buffer().unwrap().get_backing_store(); + let backing_store = view.buffer(scope).unwrap().get_backing_store(); let buf = unsafe { get_backing_store_slice( &backing_store, @@ -734,19 +766,19 @@ fn shared_getter( _args: v8::PropertyCallbackArguments, mut rv: v8::ReturnValue, ) { - let core_isolate: &mut CoreIsolate = - unsafe { &mut *(scope.isolate().get_data(0) as *mut CoreIsolate) }; + let state_rc = CoreIsolate::state(scope.isolate()); + let mut state = state_rc.borrow_mut(); // Lazily initialize the persistent external ArrayBuffer. - if core_isolate.shared_ab.is_empty() { + if state.shared_ab.is_empty() { let ab = v8::SharedArrayBuffer::with_backing_store( scope, - core_isolate.shared.get_backing_store(), + state.shared.get_backing_store(), ); - core_isolate.shared_ab.set(scope, ab); + state.shared_ab.set(scope, ab); } - let shared_ab = core_isolate.shared_ab.get(scope).unwrap(); + let shared_ab = state.shared_ab.get(scope).unwrap(); rv.set(shared_ab.into()); } @@ -759,11 +791,11 @@ pub fn module_resolve_callback<'s>( let mut scope = v8::EscapableHandleScope::new(scope.enter()); let scope = scope.enter(); - let core_isolate: &mut EsIsolate = - unsafe { &mut *(scope.isolate().get_data(1) as *mut EsIsolate) }; + let state_rc = EsIsolate::state(scope.isolate()); + let mut state = state_rc.borrow_mut(); let referrer_id = referrer.get_identity_hash(); - let referrer_name = core_isolate + let referrer_name = state .modules .get_info(referrer_id) .expect("ModuleInfo not found") @@ -778,8 +810,8 @@ pub fn module_resolve_callback<'s>( let req_str = req.to_rust_string_lossy(scope); if req_str == specifier_str { - let id = core_isolate.module_resolve_cb(&req_str, referrer_id); - let maybe_info = core_isolate.modules.get_info(id); + let id = state.module_resolve_cb(&req_str, referrer_id); + let maybe_info = state.modules.get_info(id); if maybe_info.is_none() { let msg = format!( @@ -810,10 +842,10 @@ fn get_promise_details( args: v8::FunctionCallbackArguments, mut rv: v8::ReturnValue, ) { - let core_isolate: &mut CoreIsolate = - unsafe { &mut *(scope.isolate().get_data(0) as *mut CoreIsolate) }; - assert!(!core_isolate.global_context.is_empty()); - let context = core_isolate.global_context.get(scope).unwrap(); + let state_rc = CoreIsolate::state(scope.isolate()); + let state = state_rc.borrow(); + assert!(!state.global_context.is_empty()); + let context = state.global_context.get(scope).unwrap(); let promise = match v8::Local::::try_from(args.get(0)) { Ok(val) => val, diff --git a/core/core.js b/core/core.js index 4c6f708bb1932a..23cc325abfeabe 100644 --- a/core/core.js +++ b/core/core.js @@ -59,7 +59,7 @@ SharedQueue Binary Layout function ops() { // op id 0 is a special value to retrieve the map of registered ops. - const opsMapBytes = send(0, new Uint8Array([]), null); + const opsMapBytes = send(0, new Uint8Array([])); const opsMapJson = String.fromCharCode.apply(null, opsMapBytes); return JSON.parse(opsMapJson); } @@ -181,13 +181,9 @@ SharedQueue Binary Layout } } - function dispatch(opId, control, zeroCopy = null) { - return send(opId, control, zeroCopy); - } - Object.assign(window.Deno.core, { setAsyncHandler, - dispatch, + dispatch: send, ops, // sharedQueue is private but exposed for testing. sharedQueue: { diff --git a/core/isolate.rs b/core/core_isolate.rs similarity index 81% rename from core/isolate.rs rename to core/core_isolate.rs index 13892c2d9ab18a..dc22d821adbc8a 100644 --- a/core/isolate.rs +++ b/core/core_isolate.rs @@ -7,99 +7,34 @@ use rusty_v8 as v8; -use crate::any_error::ErrBox; use crate::bindings; -use crate::js_errors::JSError; use crate::ops::*; use crate::shared_queue::SharedQueue; use crate::shared_queue::RECOMMENDED_SIZE; +use crate::ErrBox; +use crate::JSError; use crate::ResourceTable; +use crate::ZeroCopyBuf; use futures::future::FutureExt; -use futures::stream::select; use futures::stream::FuturesUnordered; use futures::stream::StreamExt; use futures::task::AtomicWaker; use futures::Future; -use libc::c_void; use std::cell::RefCell; use std::collections::HashMap; use std::convert::From; -use std::error::Error; -use std::fmt; use std::mem::forget; -use std::ops::{Deref, DerefMut}; +use std::ops::Deref; +use std::ops::DerefMut; use std::option::Option; use std::pin::Pin; use std::rc::Rc; -use std::sync::{Arc, Mutex, Once}; +use std::sync::Once; use std::task::Context; use std::task::Poll; type PendingOpFuture = Pin>>; -/// A ZeroCopyBuf encapsulates a slice that's been borrowed from a JavaScript -/// ArrayBuffer object. JavaScript objects can normally be garbage collected, -/// but the existence of a ZeroCopyBuf inhibits this until it is dropped. It -/// behaves much like an Arc<[u8]>, although a ZeroCopyBuf currently can't be -/// cloned. -pub struct ZeroCopyBuf { - backing_store: v8::SharedRef, - byte_offset: usize, - byte_length: usize, -} - -unsafe impl Send for ZeroCopyBuf {} - -impl ZeroCopyBuf { - pub fn new(view: v8::Local) -> Self { - let backing_store = view.buffer().unwrap().get_backing_store(); - let byte_offset = view.byte_offset(); - let byte_length = view.byte_length(); - Self { - backing_store, - byte_offset, - byte_length, - } - } -} - -impl Deref for ZeroCopyBuf { - type Target = [u8]; - fn deref(&self) -> &[u8] { - unsafe { - bindings::get_backing_store_slice( - &self.backing_store, - self.byte_offset, - self.byte_length, - ) - } - } -} - -impl DerefMut for ZeroCopyBuf { - fn deref_mut(&mut self) -> &mut [u8] { - unsafe { - bindings::get_backing_store_slice_mut( - &self.backing_store, - self.byte_offset, - self.byte_length, - ) - } - } -} - -impl AsRef<[u8]> for ZeroCopyBuf { - fn as_ref(&self) -> &[u8] { - &*self - } -} - -impl AsMut<[u8]> for ZeroCopyBuf { - fn as_mut(&mut self) -> &mut [u8] { - &mut *self - } -} - /// Stores a script used to initialize a Isolate pub struct Script<'a> { pub source: &'a str, @@ -137,7 +72,6 @@ pub enum StartupData<'a> { } type JSErrorCreateFn = dyn Fn(JSError) -> ErrBox; -type IsolateErrorHandleFn = dyn FnMut(ErrBox) -> Result<(), ErrBox>; /// A single execution context of JavaScript. Corresponds roughly to the "Web /// Worker" concept in the DOM. An CoreIsolate is a Future that can be used with @@ -147,28 +81,53 @@ type IsolateErrorHandleFn = dyn FnMut(ErrBox) -> Result<(), ErrBox>; /// Ops are created in JavaScript by calling Deno.core.dispatch(), and in Rust /// by implementing dispatcher function that takes control buffer and optional zero copy buffer /// as arguments. An async Op corresponds exactly to a Promise in JavaScript. -#[allow(unused)] pub struct CoreIsolate { - pub v8_isolate: Option, + // This is an Option instead of just OwnedIsolate to workaround + // an safety issue with SnapshotCreator. See CoreIsolate::drop. + v8_isolate: Option, snapshot_creator: Option, has_snapshotted: bool, + needs_init: bool, + startup_script: Option, +} + +/// Internal state for CoreIsolate which is stored in one of v8::Isolate's +/// embedder slots. +pub struct CoreIsolateState { pub resource_table: Rc>, pub global_context: v8::Global, pub(crate) shared_ab: v8::Global, pub(crate) js_recv_cb: v8::Global, pub(crate) js_macrotask_cb: v8::Global, pub(crate) pending_promise_exceptions: HashMap>, - shared_isolate_handle: Arc>>, pub(crate) js_error_create_fn: Box, - needs_init: bool, pub(crate) shared: SharedQueue, pending_ops: FuturesUnordered, pending_unref_ops: FuturesUnordered, have_unpolled_ops: bool, - startup_script: Option, pub op_registry: OpRegistry, waker: AtomicWaker, - error_handler: Option>, +} + +// TODO(ry) The trait v8::InIsolate is superfluous. HandleScope::new should just +// take &mut v8::Isolate. +impl v8::InIsolate for CoreIsolate { + fn isolate(&mut self) -> &mut v8::Isolate { + self.v8_isolate.as_mut().unwrap() + } +} + +impl Deref for CoreIsolate { + type Target = v8::Isolate; + fn deref(&self) -> &v8::Isolate { + self.v8_isolate.as_ref().unwrap() + } +} + +impl DerefMut for CoreIsolate { + fn deref_mut(&mut self) -> &mut v8::Isolate { + self.v8_isolate.as_mut().unwrap() + } } impl Drop for CoreIsolate { @@ -193,8 +152,6 @@ impl Drop for CoreIsolate { } } -static DENO_INIT: Once = Once::new(); - #[allow(clippy::missing_safety_doc)] pub unsafe fn v8_init() { let platform = v8::new_default_platform().unwrap(); @@ -215,7 +172,8 @@ pub unsafe fn v8_init() { impl CoreIsolate { /// startup_data defines the snapshot or script used at startup to initialize /// the isolate. - pub fn new(startup_data: StartupData, will_snapshot: bool) -> Box { + pub fn new(startup_data: StartupData, will_snapshot: bool) -> Self { + static DENO_INIT: Once = Once::new(); DENO_INIT.call_once(|| { unsafe { v8_init() }; }); @@ -275,44 +233,29 @@ impl CoreIsolate { (isolate, None) }; - let shared = SharedQueue::new(RECOMMENDED_SIZE); - let needs_init = true; - - let core_isolate = Self { - v8_isolate: None, + isolate.set_slot(Rc::new(RefCell::new(CoreIsolateState { global_context, resource_table: Rc::new(RefCell::new(ResourceTable::default())), pending_promise_exceptions: HashMap::new(), shared_ab: v8::Global::::new(), js_recv_cb: v8::Global::::new(), js_macrotask_cb: v8::Global::::new(), - snapshot_creator: maybe_snapshot_creator, - has_snapshotted: false, - shared_isolate_handle: Arc::new(Mutex::new(None)), js_error_create_fn: Box::new(JSError::create), - shared, - needs_init, + shared: SharedQueue::new(RECOMMENDED_SIZE), pending_ops: FuturesUnordered::new(), pending_unref_ops: FuturesUnordered::new(), have_unpolled_ops: false, - startup_script, op_registry: OpRegistry::new(), waker: AtomicWaker::new(), - error_handler: None, - }; + }))); - let mut boxed_isolate = Box::new(core_isolate); - { - let core_isolate_ptr: *mut Self = Box::into_raw(boxed_isolate); - unsafe { isolate.set_data(0, core_isolate_ptr as *mut c_void) }; - boxed_isolate = unsafe { Box::from_raw(core_isolate_ptr) }; - let shared_handle_ptr = &mut *isolate; - *boxed_isolate.shared_isolate_handle.lock().unwrap() = - Some(shared_handle_ptr); - boxed_isolate.v8_isolate = Some(isolate); + Self { + v8_isolate: Some(isolate), + snapshot_creator: maybe_snapshot_creator, + has_snapshotted: false, + needs_init: true, + startup_script, } - - boxed_isolate } fn setup_isolate(mut isolate: v8::OwnedIsolate) -> v8::OwnedIsolate { @@ -321,26 +264,9 @@ impl CoreIsolate { isolate } - /// Defines the how Deno.core.dispatch() acts. - /// Called whenever Deno.core.dispatch() is called in JavaScript. zero_copy_buf - /// corresponds to the second argument of Deno.core.dispatch(). - /// - /// Requires runtime to explicitly ask for op ids before using any of the ops. - pub fn register_op(&mut self, name: &str, op: F) -> OpId - where - F: Fn(&mut CoreIsolate, &[u8], Option) -> Op + 'static, - { - self.op_registry.register(name, op) - } - - /// Allows a callback to be set whenever a V8 exception is made. This allows - /// the caller to wrap the JSError into an error. By default this callback - /// is set to JSError::create. - pub fn set_js_error_create_fn( - &mut self, - f: impl Fn(JSError) -> ErrBox + 'static, - ) { - self.js_error_create_fn = Box::new(f); + pub fn state(isolate: &v8::Isolate) -> Rc> { + let s = isolate.get_slot::>>().unwrap(); + s.clone() } /// Executes a bit of built-in JavaScript to provide Deno.sharedQueue. @@ -355,46 +281,6 @@ impl CoreIsolate { } } - pub fn dispatch_op<'s>( - &mut self, - scope: &mut impl v8::ToLocal<'s>, - op_id: OpId, - control_buf: &[u8], - zero_copy_buf: Option, - ) -> Option<(OpId, Box<[u8]>)> { - let op = if let Some(dispatcher) = self.op_registry.get(op_id) { - dispatcher(self, control_buf, zero_copy_buf) - } else { - let message = - v8::String::new(scope, &format!("Unknown op id: {}", op_id)).unwrap(); - let exception = v8::Exception::type_error(scope, message); - scope.isolate().throw_exception(exception); - return None; - }; - - debug_assert_eq!(self.shared.size(), 0); - match op { - Op::Sync(buf) => { - // For sync messages, we always return the response via Deno.core.send's - // return value. Sync messages ignore the op_id. - let op_id = 0; - Some((op_id, buf)) - } - Op::Async(fut) => { - let fut2 = fut.map(move |buf| (op_id, buf)); - self.pending_ops.push(fut2.boxed_local()); - self.have_unpolled_ops = true; - None - } - Op::AsyncUnref(fut) => { - let fut2 = fut.map(move |buf| (op_id, buf)); - self.pending_unref_ops.push(fut2.boxed_local()); - self.have_unpolled_ops = true; - None - } - } - } - /// Executes traditional JavaScript code (traditional = not ES modules) /// /// ErrBox can be downcast to a type that exposes additional information about @@ -407,16 +293,17 @@ impl CoreIsolate { ) -> Result<(), ErrBox> { self.shared_init(); - let js_error_create_fn = &*self.js_error_create_fn; - let v8_isolate = self.v8_isolate.as_mut().unwrap(); + let state_rc = Self::state(self); + let state = state_rc.borrow(); - let mut hs = v8::HandleScope::new(v8_isolate); + let mut hs = v8::HandleScope::new(self.v8_isolate.as_mut().unwrap()); let scope = hs.enter(); - assert!(!self.global_context.is_empty()); - let context = self.global_context.get(scope).unwrap(); + let context = state.global_context.get(scope).unwrap(); let mut cs = v8::ContextScope::new(scope, context); let scope = cs.enter(); + drop(state); + let source = v8::String::new(scope, js_source).unwrap(); let name = v8::String::new(scope, js_filename).unwrap(); let origin = bindings::script_origin(scope, name); @@ -428,8 +315,8 @@ impl CoreIsolate { match v8::Script::compile(scope, context, source, Some(&origin)) { Some(script) => script, None => { - let exception = tc.exception().unwrap(); - return exception_to_err_result(scope, exception, js_error_create_fn); + let exception = tc.exception(scope).unwrap(); + return exception_to_err_result(scope, exception); } }; @@ -437,8 +324,8 @@ impl CoreIsolate { Some(_) => Ok(()), None => { assert!(tc.has_caught()); - let exception = tc.exception().unwrap(); - exception_to_err_result(scope, exception, js_error_create_fn) + let exception = tc.exception(scope).unwrap(); + exception_to_err_result(scope, exception) } } } @@ -451,6 +338,7 @@ impl CoreIsolate { /// different type if CoreIsolate::set_js_error_create_fn() has been used. pub fn snapshot(&mut self) -> v8::StartupData { assert!(self.snapshot_creator.is_some()); + let state = Self::state(self); // Note: create_blob() method must not be called from within a HandleScope. // The HandleScope created here is exited at the end of the block. @@ -459,7 +347,7 @@ impl CoreIsolate { let v8_isolate = self.v8_isolate.as_mut().unwrap(); let mut hs = v8::HandleScope::new(v8_isolate); let scope = hs.enter(); - self.global_context.reset(scope); + state.borrow_mut().global_context.reset(scope); } let snapshot_creator = self.snapshot_creator.as_mut().unwrap(); @@ -470,47 +358,59 @@ impl CoreIsolate { snapshot } + + /// Defines the how Deno.core.dispatch() acts. + /// Called whenever Deno.core.dispatch() is called in JavaScript. zero_copy_buf + /// corresponds to the second argument of Deno.core.dispatch(). + /// + /// Requires runtime to explicitly ask for op ids before using any of the ops. + pub fn register_op(&mut self, name: &str, op: F) -> OpId + where + F: Fn(&mut CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op + 'static, + { + let state_rc = Self::state(self); + let mut state = state_rc.borrow_mut(); + state.op_registry.register(name, op) + } } impl Future for CoreIsolate { type Output = Result<(), ErrBox>; fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let inner = self.get_mut(); - inner.waker.register(cx.waker()); - inner.shared_init(); + let core_isolate = self.get_mut(); + core_isolate.shared_init(); - let v8_isolate = inner.v8_isolate.as_mut().unwrap(); - let js_error_create_fn = &*inner.js_error_create_fn; - let js_recv_cb = &inner.js_recv_cb; - let js_macrotask_cb = &inner.js_macrotask_cb; - let pending_promise_exceptions = &mut inner.pending_promise_exceptions; + let state_rc = Self::state(core_isolate); + { + let state = state_rc.borrow(); + state.waker.register(cx.waker()); + } - let mut hs = v8::HandleScope::new(v8_isolate); + let mut hs = v8::HandleScope::new(core_isolate); let scope = hs.enter(); - let context = inner.global_context.get(scope).unwrap(); + let context = { + let state = state_rc.borrow(); + state.global_context.get(scope).unwrap() + }; let mut cs = v8::ContextScope::new(scope, context); let scope = cs.enter(); - check_promise_exceptions( - scope, - pending_promise_exceptions, - js_error_create_fn, - )?; + check_promise_exceptions(scope)?; let mut overflow_response: Option<(OpId, Buf)> = None; loop { + let mut state = state_rc.borrow_mut(); // Now handle actual ops. - inner.have_unpolled_ops = false; - #[allow(clippy::match_wild_err_arm)] - match select(&mut inner.pending_ops, &mut inner.pending_unref_ops) - .poll_next_unpin(cx) - { + state.have_unpolled_ops = false; + + let pending_r = state.pending_ops.poll_next_unpin(cx); + match pending_r { Poll::Ready(None) => break, Poll::Pending => break, Poll::Ready(Some((op_id, buf))) => { - let successful_push = inner.shared.push(op_id, &buf); + let successful_push = state.shared.push(op_id, &buf); if !successful_push { // If we couldn't push the response to the shared queue, because // there wasn't enough size, we will return the buffer via the @@ -519,55 +419,142 @@ impl Future for CoreIsolate { break; } } - } + }; } - if inner.shared.size() > 0 { - async_op_response(scope, None, js_recv_cb, js_error_create_fn)?; - // The other side should have shifted off all the messages. - assert_eq!(inner.shared.size(), 0); + loop { + let mut state = state_rc.borrow_mut(); + let unref_r = state.pending_unref_ops.poll_next_unpin(cx); + #[allow(clippy::match_wild_err_arm)] + match unref_r { + Poll::Ready(None) => break, + Poll::Pending => break, + Poll::Ready(Some((op_id, buf))) => { + let successful_push = state.shared.push(op_id, &buf); + if !successful_push { + // If we couldn't push the response to the shared queue, because + // there wasn't enough size, we will return the buffer via the + // legacy route, using the argument of deno_respond. + overflow_response = Some((op_id, buf)); + break; + } + } + }; } - if let Some((op_id, buf)) = overflow_response.take() { - async_op_response( - scope, - Some((op_id, buf)), - js_recv_cb, - js_error_create_fn, - )?; + { + let state = state_rc.borrow(); + if state.shared.size() > 0 { + drop(state); + async_op_response(scope, None)?; + // The other side should have shifted off all the messages. + let state = state_rc.borrow(); + assert_eq!(state.shared.size(), 0); + } } - drain_macrotasks(scope, js_macrotask_cb, js_error_create_fn)?; + { + if let Some((op_id, buf)) = overflow_response.take() { + async_op_response(scope, Some((op_id, buf)))?; + } - check_promise_exceptions( - scope, - pending_promise_exceptions, - js_error_create_fn, - )?; + drain_macrotasks(scope)?; + + check_promise_exceptions(scope)?; + } + let state = state_rc.borrow(); // We're idle if pending_ops is empty. - if inner.pending_ops.is_empty() { + if state.pending_ops.is_empty() { Poll::Ready(Ok(())) } else { - if inner.have_unpolled_ops { - inner.waker.wake(); + if state.have_unpolled_ops { + state.waker.wake(); } Poll::Pending } } } +impl CoreIsolateState { + /// Defines the how Deno.core.dispatch() acts. + /// Called whenever Deno.core.dispatch() is called in JavaScript. zero_copy_buf + /// corresponds to the second argument of Deno.core.dispatch(). + /// + /// Requires runtime to explicitly ask for op ids before using any of the ops. + pub fn register_op(&mut self, name: &str, op: F) -> OpId + where + F: Fn(&mut CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op + 'static, + { + self.op_registry.register(name, op) + } + + /// Allows a callback to be set whenever a V8 exception is made. This allows + /// the caller to wrap the JSError into an error. By default this callback + /// is set to JSError::create. + pub fn set_js_error_create_fn( + &mut self, + f: impl Fn(JSError) -> ErrBox + 'static, + ) { + self.js_error_create_fn = Box::new(f); + } + + pub fn dispatch_op<'s>( + &mut self, + scope: &mut impl v8::ToLocal<'s>, + op_id: OpId, + control_buf: &[u8], + zero_copy_bufs: &mut [ZeroCopyBuf], + ) -> Option<(OpId, Box<[u8]>)> { + let op = if let Some(dispatcher) = self.op_registry.get(op_id) { + dispatcher(self, control_buf, zero_copy_bufs) + } else { + let message = + v8::String::new(scope, &format!("Unknown op id: {}", op_id)).unwrap(); + let exception = v8::Exception::type_error(scope, message); + scope.isolate().throw_exception(exception); + return None; + }; + + debug_assert_eq!(self.shared.size(), 0); + match op { + Op::Sync(buf) => { + // For sync messages, we always return the response via Deno.core.send's + // return value. Sync messages ignore the op_id. + let op_id = 0; + Some((op_id, buf)) + } + Op::Async(fut) => { + let fut2 = fut.map(move |buf| (op_id, buf)); + self.pending_ops.push(fut2.boxed_local()); + self.have_unpolled_ops = true; + None + } + Op::AsyncUnref(fut) => { + let fut2 = fut.map(move |buf| (op_id, buf)); + self.pending_unref_ops.push(fut2.boxed_local()); + self.have_unpolled_ops = true; + None + } + } + } +} + fn async_op_response<'s>( scope: &mut impl v8::ToLocal<'s>, maybe_buf: Option<(OpId, Box<[u8]>)>, - js_recv_cb: &v8::Global, - js_error_create_fn: &JSErrorCreateFn, ) -> Result<(), ErrBox> { let context = scope.get_current_context().unwrap(); let global: v8::Local = context.global(scope).into(); - let js_recv_cb = js_recv_cb + + let state_rc = CoreIsolate::state(scope.isolate()); + let state = state_rc.borrow_mut(); + + let js_recv_cb = state + .js_recv_cb .get(scope) .expect("Deno.core.recv has not been called."); + drop(state); // TODO(piscisaureus): properly integrate TryCatch in the scope chain. let mut try_catch = v8::TryCatch::new(scope); @@ -584,22 +571,23 @@ fn async_op_response<'s>( None => js_recv_cb.call(scope, context, global, &[]), }; - match tc.exception() { + match tc.exception(scope) { None => Ok(()), - Some(exception) => { - exception_to_err_result(scope, exception, js_error_create_fn) - } + Some(exception) => exception_to_err_result(scope, exception), } } fn drain_macrotasks<'s>( scope: &mut impl v8::ToLocal<'s>, - js_macrotask_cb: &v8::Global, - js_error_create_fn: &JSErrorCreateFn, ) -> Result<(), ErrBox> { let context = scope.get_current_context().unwrap(); let global: v8::Local = context.global(scope).into(); - let js_macrotask_cb = js_macrotask_cb.get(scope); + + let js_macrotask_cb = { + let state_rc = CoreIsolate::state(scope.isolate()); + let state = state_rc.borrow_mut(); + state.js_macrotask_cb.get(scope) + }; if js_macrotask_cb.is_none() { return Ok(()); } @@ -614,8 +602,8 @@ fn drain_macrotasks<'s>( let is_done = js_macrotask_cb.call(scope, context, global, &[]); - if let Some(exception) = tc.exception() { - return exception_to_err_result(scope, exception, js_error_create_fn); + if let Some(exception) = tc.exception(scope) { + return exception_to_err_result(scope, exception); } let is_done = is_done.unwrap(); @@ -627,18 +615,9 @@ fn drain_macrotasks<'s>( Ok(()) } -pub(crate) fn attach_handle_to_error( - scope: &mut impl v8::InIsolate, - err: ErrBox, - handle: v8::Local, -) -> ErrBox { - ErrWithV8Handle::new(scope, err, handle).into() -} - pub(crate) fn exception_to_err_result<'s, T>( scope: &mut impl v8::ToLocal<'s>, exception: v8::Local, - js_error_create_fn: &JSErrorCreateFn, ) -> Result { // TODO(piscisaureus): in rusty_v8, `is_execution_terminating()` should // also be implemented on `struct Isolate`. @@ -666,7 +645,10 @@ pub(crate) fn exception_to_err_result<'s, T>( } let js_error = JSError::from_v8_exception(scope, exception); - let js_error = (js_error_create_fn)(js_error); + + let state_rc = CoreIsolate::state(scope.isolate()); + let state = state_rc.borrow(); + let js_error = (state.js_error_create_fn)(js_error); if is_terminating_exception { // Re-enable exception termination. @@ -680,13 +662,15 @@ pub(crate) fn exception_to_err_result<'s, T>( fn check_promise_exceptions<'s>( scope: &mut impl v8::ToLocal<'s>, - pending_promise_exceptions: &mut HashMap>, - js_error_create_fn: &JSErrorCreateFn, ) -> Result<(), ErrBox> { - if let Some(&key) = pending_promise_exceptions.keys().next() { - let handle = pending_promise_exceptions.remove(&key).unwrap(); + let state_rc = CoreIsolate::state(scope.isolate()); + let mut state = state_rc.borrow_mut(); + + if let Some(&key) = state.pending_promise_exceptions.keys().next() { + let handle = state.pending_promise_exceptions.remove(&key).unwrap(); + drop(state); let exception = handle.get(scope).expect("empty error handle"); - exception_to_err_result(scope, exception, js_error_create_fn) + exception_to_err_result(scope, exception) } else { Ok(()) } @@ -705,6 +689,7 @@ pub mod tests { use futures::future::lazy; use std::ops::FnOnce; use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; pub fn run_in_task(f: F) where @@ -733,21 +718,22 @@ pub mod tests { pub enum Mode { Async, AsyncUnref, + AsyncZeroCopy(u8), OverflowReqSync, OverflowResSync, OverflowReqAsync, OverflowResAsync, } - pub fn setup(mode: Mode) -> (Box, Arc) { + pub fn setup(mode: Mode) -> (CoreIsolate, Arc) { let dispatch_count = Arc::new(AtomicUsize::new(0)); let dispatch_count_ = dispatch_count.clone(); let mut isolate = CoreIsolate::new(StartupData::None, false); - let dispatcher = move |_isolate: &mut CoreIsolate, + let dispatcher = move |_state: &mut CoreIsolateState, control: &[u8], - _zero_copy: Option| + zero_copy: &mut [ZeroCopyBuf]| -> Op { dispatch_count_.fetch_add(1, Ordering::Relaxed); match mode { @@ -767,6 +753,18 @@ pub mod tests { }; Op::AsyncUnref(fut.boxed()) } + Mode::AsyncZeroCopy(count) => { + assert_eq!(control.len(), 1); + assert_eq!(control[0], 24); + assert_eq!(zero_copy.len(), count as usize); + zero_copy.iter().enumerate().for_each(|(idx, buf)| { + assert_eq!(buf.len(), 1); + assert_eq!(idx, buf[0] as usize); + }); + + let buf = vec![43u8].into_boxed_slice(); + Op::Async(futures::future::ready(buf).boxed()) + } Mode::OverflowReqSync => { assert_eq!(control.len(), 100 * 1024 * 1024); let buf = vec![43u8].into_boxed_slice(); @@ -831,6 +829,48 @@ pub mod tests { assert_eq!(dispatch_count.load(Ordering::Relaxed), 2); } + #[test] + fn test_dispatch_no_zero_copy_buf() { + let (mut isolate, dispatch_count) = setup(Mode::AsyncZeroCopy(0)); + js_check(isolate.execute( + "filename.js", + r#" + let control = new Uint8Array([24]); + Deno.core.send(1, control); + "#, + )); + assert_eq!(dispatch_count.load(Ordering::Relaxed), 1); + } + + #[test] + fn test_dispatch_one_zero_copy_buf() { + let (mut isolate, dispatch_count) = setup(Mode::AsyncZeroCopy(1)); + js_check(isolate.execute( + "filename.js", + r#" + let control = new Uint8Array([24]); + let zero_copy = new Uint8Array([0]); + Deno.core.send(1, control, zero_copy); + "#, + )); + assert_eq!(dispatch_count.load(Ordering::Relaxed), 1); + } + + #[test] + fn test_dispatch_two_zero_copy_bufs() { + let (mut isolate, dispatch_count) = setup(Mode::AsyncZeroCopy(2)); + js_check(isolate.execute( + "filename.js", + r#" + let control = new Uint8Array([24]); + let zero_copy_a = new Uint8Array([0]); + let zero_copy_b = new Uint8Array([1]); + Deno.core.send(1, control, zero_copy_a, zero_copy_b); + "#, + )); + assert_eq!(dispatch_count.load(Ordering::Relaxed), 1); + } + #[test] fn test_poll_async_delayed_ops() { run_in_task(|cx| { @@ -1183,42 +1223,3 @@ pub mod tests { js_check(isolate2.execute("check.js", "if (a != 3) throw Error('x')")); } } - -// TODO(piscisaureus): rusty_v8 should implement the Error trait on -// values of type v8::Global. -pub struct ErrWithV8Handle { - err: ErrBox, - handle: v8::Global, -} - -impl ErrWithV8Handle { - pub fn new( - scope: &mut impl v8::InIsolate, - err: ErrBox, - handle: v8::Local, - ) -> Self { - let handle = v8::Global::new_from(scope, handle); - Self { err, handle } - } - - pub fn get_handle(&self) -> &v8::Global { - &self.handle - } -} - -unsafe impl Send for ErrWithV8Handle {} -unsafe impl Sync for ErrWithV8Handle {} - -impl Error for ErrWithV8Handle {} - -impl fmt::Display for ErrWithV8Handle { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.err.fmt(f) - } -} - -impl fmt::Debug for ErrWithV8Handle { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.err.fmt(f) - } -} diff --git a/core/js_errors.rs b/core/errors.rs similarity index 81% rename from core/js_errors.rs rename to core/errors.rs index e8ea5a342f529a..bc821b266fdc51 100644 --- a/core/js_errors.rs +++ b/core/errors.rs @@ -1,11 +1,76 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -use crate::ErrBox; use rusty_v8 as v8; +use std::any::Any; +use std::any::TypeId; use std::convert::TryFrom; use std::convert::TryInto; use std::error::Error; use std::fmt; +use std::ops::Deref; + +// The Send and Sync traits are required because deno is multithreaded and we +// need to be able to handle errors across threads. +pub trait AnyError: Any + Error + Send + Sync + 'static {} +impl AnyError for T where T: Any + Error + Send + Sync + Sized + 'static {} + +#[derive(Debug)] +pub struct ErrBox(Box); + +impl dyn AnyError { + pub fn downcast_ref(&self) -> Option<&T> { + if Any::type_id(self) == TypeId::of::() { + let target = self as *const Self as *const T; + let target = unsafe { &*target }; + Some(target) + } else { + None + } + } +} + +impl ErrBox { + pub fn downcast(self) -> Result { + if Any::type_id(&*self.0) == TypeId::of::() { + let target = Box::into_raw(self.0) as *mut T; + let target = unsafe { Box::from_raw(target) }; + Ok(*target) + } else { + Err(self) + } + } +} + +impl AsRef for ErrBox { + fn as_ref(&self) -> &dyn AnyError { + self.0.as_ref() + } +} + +impl Deref for ErrBox { + type Target = Box; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From for ErrBox { + fn from(error: T) -> Self { + Self(Box::new(error)) + } +} + +impl From> for ErrBox { + fn from(boxed: Box) -> Self { + Self(boxed) + } +} + +impl fmt::Display for ErrBox { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} /// A `JSError` represents an exception coming from V8, with stack frames and /// line numbers. The deno_cli crate defines another `JSError` type, which wraps @@ -298,3 +363,50 @@ impl fmt::Display for JSError { Ok(()) } } + +pub(crate) fn attach_handle_to_error( + scope: &mut impl v8::InIsolate, + err: ErrBox, + handle: v8::Local, +) -> ErrBox { + ErrWithV8Handle::new(scope, err, handle).into() +} + +// TODO(piscisaureus): rusty_v8 should implement the Error trait on +// values of type v8::Global. +pub struct ErrWithV8Handle { + err: ErrBox, + handle: v8::Global, +} + +impl ErrWithV8Handle { + pub fn new( + scope: &mut impl v8::InIsolate, + err: ErrBox, + handle: v8::Local, + ) -> Self { + let handle = v8::Global::new_from(scope, handle); + Self { err, handle } + } + + pub fn get_handle(&self) -> &v8::Global { + &self.handle + } +} + +unsafe impl Send for ErrWithV8Handle {} +unsafe impl Sync for ErrWithV8Handle {} + +impl Error for ErrWithV8Handle {} + +impl fmt::Display for ErrWithV8Handle { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.err.fmt(f) + } +} + +impl fmt::Debug for ErrWithV8Handle { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.err.fmt(f) + } +} diff --git a/core/es_isolate.rs b/core/es_isolate.rs index 73ff1c388dce7c..c778de1ccceb43 100644 --- a/core/es_isolate.rs +++ b/core/es_isolate.rs @@ -6,17 +6,17 @@ use rusty_v8 as v8; -use crate::any_error::ErrBox; use crate::bindings; +use crate::errors::ErrBox; +use crate::errors::ErrWithV8Handle; use crate::futures::FutureExt; -use crate::ErrWithV8Handle; use futures::ready; use futures::stream::FuturesUnordered; use futures::stream::StreamExt; use futures::stream::StreamFuture; use futures::task::AtomicWaker; use futures::Future; -use libc::c_void; +use std::cell::RefCell; use std::collections::HashMap; use std::convert::TryFrom; use std::ops::{Deref, DerefMut}; @@ -26,20 +26,19 @@ use std::rc::Rc; use std::task::Context; use std::task::Poll; -use crate::isolate::attach_handle_to_error; -use crate::isolate::exception_to_err_result; -use crate::isolate::CoreIsolate; -use crate::isolate::StartupData; +use crate::core_isolate::exception_to_err_result; +use crate::errors::attach_handle_to_error; use crate::module_specifier::ModuleSpecifier; use crate::modules::LoadState; +use crate::modules::ModuleId; +use crate::modules::ModuleLoadId; use crate::modules::ModuleLoader; use crate::modules::ModuleSource; use crate::modules::Modules; use crate::modules::PrepareLoadFuture; use crate::modules::RecursiveModuleLoad; - -pub type ModuleId = i32; -pub type ModuleLoadId = i32; +use crate::CoreIsolate; +use crate::StartupData; /// More specialized version of `CoreIsolate` that provides loading /// and execution of ES Modules. @@ -47,8 +46,9 @@ pub type ModuleLoadId = i32; /// Creating `EsIsolate` requires to pass `loader` argument /// that implements `ModuleLoader` trait - that way actual resolution and /// loading of modules can be customized by the implementor. -pub struct EsIsolate { - core_isolate: Box, +pub struct EsIsolate(CoreIsolate); + +pub struct EsIsolateState { loader: Rc, pub modules: Modules, pub(crate) dyn_import_map: @@ -63,13 +63,13 @@ impl Deref for EsIsolate { type Target = CoreIsolate; fn deref(&self) -> &Self::Target { - &self.core_isolate + &self.0 } } impl DerefMut for EsIsolate { fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.core_isolate + &mut self.0 } } @@ -78,38 +78,27 @@ impl EsIsolate { loader: Rc, startup_data: StartupData, will_snapshot: bool, - ) -> Box { + ) -> Self { let mut core_isolate = CoreIsolate::new(startup_data, will_snapshot); { - let v8_isolate = core_isolate.v8_isolate.as_mut().unwrap(); - v8_isolate.set_host_initialize_import_meta_object_callback( + core_isolate.set_host_initialize_import_meta_object_callback( bindings::host_initialize_import_meta_object_callback, ); - v8_isolate.set_host_import_module_dynamically_callback( + core_isolate.set_host_import_module_dynamically_callback( bindings::host_import_module_dynamically_callback, ); } - let es_isolate = Self { + core_isolate.set_slot(Rc::new(RefCell::new(EsIsolateState { modules: Modules::new(), loader, - core_isolate, dyn_import_map: HashMap::new(), preparing_dyn_imports: FuturesUnordered::new(), pending_dyn_imports: FuturesUnordered::new(), waker: AtomicWaker::new(), - }; + }))); - let mut boxed_es_isolate = Box::new(es_isolate); - { - let es_isolate_ptr: *mut Self = Box::into_raw(boxed_es_isolate); - boxed_es_isolate = unsafe { Box::from_raw(es_isolate_ptr) }; - unsafe { - let v8_isolate = boxed_es_isolate.v8_isolate.as_mut().unwrap(); - v8_isolate.set_data(1, es_isolate_ptr as *mut c_void); - }; - } - boxed_es_isolate + EsIsolate(core_isolate) } /// Low-level module creation. @@ -121,14 +110,13 @@ impl EsIsolate { name: &str, source: &str, ) -> Result { - let core_isolate = &mut self.core_isolate; - let v8_isolate = core_isolate.v8_isolate.as_mut().unwrap(); - let js_error_create_fn = &*core_isolate.js_error_create_fn; + let state_rc = Self::state(self); - let mut hs = v8::HandleScope::new(v8_isolate); + let core_state_rc = CoreIsolate::state(self); + let core_state = core_state_rc.borrow(); + let mut hs = v8::HandleScope::new(&mut self.0); let scope = hs.enter(); - assert!(!core_isolate.global_context.is_empty()); - let context = core_isolate.global_context.get(scope).unwrap(); + let context = core_state.global_context.get(scope).unwrap(); let mut cs = v8::ContextScope::new(scope, context); let scope = cs.enter(); @@ -145,11 +133,8 @@ impl EsIsolate { if tc.has_caught() { assert!(maybe_module.is_none()); - return exception_to_err_result( - scope, - tc.exception().unwrap(), - js_error_create_fn, - ); + let e = tc.exception(scope).unwrap(); + return exception_to_err_result(scope, e); } let module = maybe_module.unwrap(); @@ -159,16 +144,21 @@ impl EsIsolate { for i in 0..module.get_module_requests_length() { let import_specifier = module.get_module_request(i).to_rust_string_lossy(scope); + let state = state_rc.borrow(); let module_specifier = - self.loader.resolve(&import_specifier, name, false)?; + state.loader.resolve(&import_specifier, name, false)?; import_specifiers.push(module_specifier); } let mut handle = v8::Global::::new(); handle.set(scope, module); - self - .modules - .register(id, name, main, handle, import_specifiers); + + { + let mut state = state_rc.borrow_mut(); + state + .modules + .register(id, name, main, handle, import_specifiers); + } Ok(id) } @@ -178,32 +168,30 @@ impl EsIsolate { /// the V8 exception. By default this type is JSError, however it may be a /// different type if CoreIsolate::set_js_error_create_fn() has been used. fn mod_instantiate(&mut self, id: ModuleId) -> Result<(), ErrBox> { - let v8_isolate = self.core_isolate.v8_isolate.as_mut().unwrap(); - let js_error_create_fn = &*self.core_isolate.js_error_create_fn; + let state_rc = Self::state(self); + let state = state_rc.borrow(); - let mut hs = v8::HandleScope::new(v8_isolate); + let core_state_rc = CoreIsolate::state(self); + let core_state = core_state_rc.borrow(); + let mut hs = v8::HandleScope::new(&mut self.0); let scope = hs.enter(); - assert!(!self.core_isolate.global_context.is_empty()); - let context = self.core_isolate.global_context.get(scope).unwrap(); + let context = core_state.global_context.get(scope).unwrap(); let mut cs = v8::ContextScope::new(scope, context); let scope = cs.enter(); let mut try_catch = v8::TryCatch::new(scope); let tc = try_catch.enter(); - let module_info = match self.modules.get_info(id) { + let module_info = match state.modules.get_info(id) { Some(info) => info, None if id == 0 => return Ok(()), _ => panic!("module id {} not found in module table", id), }; let mut module = module_info.handle.get(scope).unwrap(); + drop(state); if module.get_status() == v8::ModuleStatus::Errored { - exception_to_err_result( - scope, - module.get_exception(), - js_error_create_fn, - )? + exception_to_err_result(scope, module.get_exception())? } let result = @@ -211,8 +199,8 @@ impl EsIsolate { match result { Some(_) => Ok(()), None => { - let exception = tc.exception().unwrap(); - exception_to_err_result(scope, exception, js_error_create_fn) + let exception = tc.exception(scope).unwrap(); + exception_to_err_result(scope, exception) } } } @@ -223,20 +211,25 @@ impl EsIsolate { /// the V8 exception. By default this type is JSError, however it may be a /// different type if CoreIsolate::set_js_error_create_fn() has been used. pub fn mod_evaluate(&mut self, id: ModuleId) -> Result<(), ErrBox> { - let core_isolate = &mut self.core_isolate; - let v8_isolate = core_isolate.v8_isolate.as_mut().unwrap(); - let js_error_create_fn = &*core_isolate.js_error_create_fn; + self.shared_init(); + let state_rc = Self::state(self); + let state = state_rc.borrow(); - let mut hs = v8::HandleScope::new(v8_isolate); + let core_state_rc = CoreIsolate::state(self); + + let mut hs = v8::HandleScope::new(&mut self.0); let scope = hs.enter(); - assert!(!core_isolate.global_context.is_empty()); - let context = core_isolate.global_context.get(scope).unwrap(); + let context = { + let core_state = core_state_rc.borrow(); + core_state.global_context.get(scope).unwrap() + }; let mut cs = v8::ContextScope::new(scope, context); let scope = cs.enter(); - let info = self.modules.get_info(id).expect("ModuleInfo not found"); + let info = state.modules.get_info(id).expect("ModuleInfo not found"); let module = info.handle.get(scope).expect("Empty module handle"); let mut status = module.get_status(); + drop(state); if status == v8::ModuleStatus::Instantiated { // IMPORTANT: Top-level-await is enabled, which means that return value // of module evaluation is a promise. @@ -267,8 +260,9 @@ impl EsIsolate { let promise = v8::Local::::try_from(value) .expect("Expected to get promise as module evaluation result"); let promise_id = promise.get_identity_hash(); + let mut core_state = core_state_rc.borrow_mut(); if let Some(mut handle) = - core_isolate.pending_promise_exceptions.remove(&promise_id) + core_state.pending_promise_exceptions.remove(&promise_id) { handle.reset(scope); } @@ -281,68 +275,41 @@ impl EsIsolate { v8::ModuleStatus::Evaluated => Ok(()), v8::ModuleStatus::Errored => { let exception = module.get_exception(); - exception_to_err_result(scope, exception, js_error_create_fn) + exception_to_err_result(scope, exception) .map_err(|err| attach_handle_to_error(scope, err, exception)) } other => panic!("Unexpected module status {:?}", other), } } - // Called by V8 during `Isolate::mod_instantiate`. - pub fn module_resolve_cb( - &mut self, - specifier: &str, - referrer_id: ModuleId, - ) -> ModuleId { - let referrer = self.modules.get_name(referrer_id).unwrap(); - let specifier = self - .loader - .resolve(specifier, referrer, false) - .expect("Module should have been already resolved"); - self.modules.get_id(specifier.as_str()).unwrap_or(0) - } - - // Called by V8 during `Isolate::mod_instantiate`. - pub fn dyn_import_cb( - &mut self, - resolver_handle: v8::Global, - specifier: &str, - referrer: &str, - ) { - debug!("dyn_import specifier {} referrer {} ", specifier, referrer); - - let load = RecursiveModuleLoad::dynamic_import( - specifier, - referrer, - self.loader.clone(), - ); - self.dyn_import_map.insert(load.id, resolver_handle); - self.waker.wake(); - let fut = load.prepare().boxed_local(); - self.preparing_dyn_imports.push(fut); - } - fn dyn_import_error( &mut self, id: ModuleLoadId, err: ErrBox, ) -> Result<(), ErrBox> { - let core_isolate = &mut self.core_isolate; - let v8_isolate = core_isolate.v8_isolate.as_mut().unwrap(); + let state_rc = Self::state(self); + let mut state = state_rc.borrow_mut(); + + let core_state_rc = CoreIsolate::state(self); + let core_state = core_state_rc.borrow(); - let mut hs = v8::HandleScope::new(v8_isolate); + let mut hs = v8::HandleScope::new(&mut self.0); let scope = hs.enter(); - let context = core_isolate.global_context.get(scope).unwrap(); + let context = core_state.global_context.get(scope).unwrap(); let mut cs = v8::ContextScope::new(scope, context); let scope = cs.enter(); - let mut resolver_handle = self + drop(core_state); + + let mut resolver_handle = state .dyn_import_map .remove(&id) .expect("Invalid dyn import id"); let resolver = resolver_handle.get(scope).unwrap(); resolver_handle.reset(scope); + drop(state); + let exception = err .downcast_ref::() .and_then(|err| err.get_handle().get(scope)) @@ -362,29 +329,42 @@ impl EsIsolate { id: ModuleLoadId, mod_id: ModuleId, ) -> Result<(), ErrBox> { + let state_rc = Self::state(self); + + let core_state_rc = CoreIsolate::state(self); + debug!("dyn_import_done {} {:?}", id, mod_id); assert!(mod_id != 0); - let v8_isolate = self.core_isolate.v8_isolate.as_mut().unwrap(); - let mut hs = v8::HandleScope::new(v8_isolate); + let mut hs = v8::HandleScope::new(&mut self.0); let scope = hs.enter(); - assert!(!self.core_isolate.global_context.is_empty()); - let context = self.core_isolate.global_context.get(scope).unwrap(); + let context = { + let core_state = core_state_rc.borrow(); + core_state.global_context.get(scope).unwrap() + }; let mut cs = v8::ContextScope::new(scope, context); let scope = cs.enter(); - let mut resolver_handle = self - .dyn_import_map - .remove(&id) - .expect("Invalid dyn import id"); + let mut resolver_handle = { + let mut state = state_rc.borrow_mut(); + state + .dyn_import_map + .remove(&id) + .expect("Invalid dyn import id") + }; let resolver = resolver_handle.get(scope).unwrap(); resolver_handle.reset(scope); - let info = self - .modules - .get_info(mod_id) - .expect("Dyn import module info not found"); - // Resolution success - let mut module = info.handle.get(scope).unwrap(); + + let module = { + let state = state_rc.borrow(); + let info = state + .modules + .get_info(mod_id) + .expect("Dyn import module info not found"); + // Resolution success + info.handle.get(scope).unwrap() + }; assert_eq!(module.get_status(), v8::ModuleStatus::Evaluated); + let module_namespace = module.get_module_namespace(); resolver.resolve(context, module_namespace).unwrap(); scope.isolate().run_microtasks(); @@ -395,8 +375,14 @@ impl EsIsolate { &mut self, cx: &mut Context, ) -> Poll> { + let state_rc = Self::state(self); + loop { - match self.preparing_dyn_imports.poll_next_unpin(cx) { + let r = { + let mut state = state_rc.borrow_mut(); + state.preparing_dyn_imports.poll_next_unpin(cx) + }; + match r { Poll::Pending | Poll::Ready(None) => { // There are no active dynamic import loaders, or none are ready. return Poll::Ready(Ok(())); @@ -407,7 +393,8 @@ impl EsIsolate { match prepare_result { Ok(load) => { - self.pending_dyn_imports.push(load.into_future()); + let state = state_rc.borrow_mut(); + state.pending_dyn_imports.push(load.into_future()); } Err(err) => { self.dyn_import_error(dyn_import_id, err)?; @@ -419,8 +406,14 @@ impl EsIsolate { } fn poll_dyn_imports(&mut self, cx: &mut Context) -> Poll> { + let state_rc = Self::state(self); loop { - match self.pending_dyn_imports.poll_next_unpin(cx) { + let poll_result = { + let mut state = state_rc.borrow_mut(); + state.pending_dyn_imports.poll_next_unpin(cx) + }; + + match poll_result { Poll::Pending | Poll::Ready(None) => { // There are no active dynamic import loaders, or none are ready. return Poll::Ready(Ok(())); @@ -439,7 +432,8 @@ impl EsIsolate { match self.register_during_load(info, &mut load) { Ok(()) => { // Keep importing until it's fully drained - self.pending_dyn_imports.push(load.into_future()); + let state = state_rc.borrow_mut(); + state.pending_dyn_imports.push(load.into_future()); } Err(err) => self.dyn_import_error(dyn_import_id, err)?, } @@ -482,6 +476,7 @@ impl EsIsolate { let referrer_specifier = ModuleSpecifier::resolve_url(&module_url_found).unwrap(); + let state_rc = Self::state(self); // #A There are 3 cases to handle at this moment: // 1. Source code resolved result have the same module name as requested // and is not yet registered @@ -494,10 +489,18 @@ impl EsIsolate { // If necessary, register an alias. if module_url_specified != module_url_found { - self.modules.alias(&module_url_specified, &module_url_found); + let mut state = state_rc.borrow_mut(); + state + .modules + .alias(&module_url_specified, &module_url_found); } - let module_id = match self.modules.get_id(&module_url_found) { + let maybe_mod_id = { + let state = state_rc.borrow(); + state.modules.get_id(&module_url_found) + }; + + let module_id = match maybe_mod_id { Some(id) => { // Module has already been registered. debug!( @@ -511,10 +514,19 @@ impl EsIsolate { }; // Now we must iterate over all imports of the module and load them. - let imports = self.modules.get_children(module_id).unwrap(); + let imports = { + let state_rc = Self::state(self); + let state = state_rc.borrow(); + state.modules.get_children(module_id).unwrap().clone() + }; for module_specifier in imports { - if !self.modules.is_registered(module_specifier) { + let is_registered = { + let state_rc = Self::state(self); + let state = state_rc.borrow(); + state.modules.is_registered(&module_specifier) + }; + if !is_registered { load .add_import(module_specifier.to_owned(), referrer_specifier.clone()); } @@ -542,11 +554,14 @@ impl EsIsolate { specifier: &ModuleSpecifier, code: Option, ) -> Result { - let load = RecursiveModuleLoad::main( - &specifier.to_string(), - code, - self.loader.clone(), - ); + self.shared_init(); + let loader = { + let state_rc = Self::state(self); + let state = state_rc.borrow(); + state.loader.clone() + }; + + let load = RecursiveModuleLoad::main(&specifier.to_string(), code, loader); let (_load_id, prepare_result) = load.prepare().await; let mut load = prepare_result?; @@ -559,30 +574,55 @@ impl EsIsolate { let root_id = load.root_module_id.expect("Root module id empty"); self.mod_instantiate(root_id).map(|_| root_id) } + + pub fn snapshot(&mut self) -> v8::StartupData { + let state_rc = Self::state(self); + std::mem::take(&mut state_rc.borrow_mut().modules); + CoreIsolate::snapshot(self) + } + + pub fn state(isolate: &v8::Isolate) -> Rc> { + let s = isolate.get_slot::>>().unwrap(); + s.clone() + } } impl Future for EsIsolate { type Output = Result<(), ErrBox>; fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let inner = self.get_mut(); + let es_isolate = self.get_mut(); - inner.waker.register(cx.waker()); + let state_rc = Self::state(es_isolate); - if !inner.preparing_dyn_imports.is_empty() { - let poll_imports = inner.prepare_dyn_imports(cx)?; + { + let state = state_rc.borrow(); + state.waker.register(cx.waker()); + } + + let has_preparing = { + let state = state_rc.borrow(); + !state.preparing_dyn_imports.is_empty() + }; + if has_preparing { + let poll_imports = es_isolate.prepare_dyn_imports(cx)?; assert!(poll_imports.is_ready()); } - if !inner.pending_dyn_imports.is_empty() { - let poll_imports = inner.poll_dyn_imports(cx)?; + let has_pending = { + let state = state_rc.borrow(); + !state.pending_dyn_imports.is_empty() + }; + if has_pending { + let poll_imports = es_isolate.poll_dyn_imports(cx)?; assert!(poll_imports.is_ready()); } - match ready!(inner.core_isolate.poll_unpin(cx)) { + match ready!(es_isolate.0.poll_unpin(cx)) { Ok(()) => { - if inner.pending_dyn_imports.is_empty() - && inner.preparing_dyn_imports.is_empty() + let state = state_rc.borrow(); + if state.pending_dyn_imports.is_empty() + && state.preparing_dyn_imports.is_empty() { Poll::Ready(Ok(())) } else { @@ -594,21 +634,58 @@ impl Future for EsIsolate { } } +impl EsIsolateState { + // Called by V8 during `Isolate::mod_instantiate`. + pub fn module_resolve_cb( + &mut self, + specifier: &str, + referrer_id: ModuleId, + ) -> ModuleId { + let referrer = self.modules.get_name(referrer_id).unwrap(); + let specifier = self + .loader + .resolve(specifier, referrer, false) + .expect("Module should have been already resolved"); + self.modules.get_id(specifier.as_str()).unwrap_or(0) + } + + // Called by V8 during `Isolate::mod_instantiate`. + pub fn dyn_import_cb( + &mut self, + resolver_handle: v8::Global, + specifier: &str, + referrer: &str, + ) { + debug!("dyn_import specifier {} referrer {} ", specifier, referrer); + + let load = RecursiveModuleLoad::dynamic_import( + specifier, + referrer, + self.loader.clone(), + ); + self.dyn_import_map.insert(load.id, resolver_handle); + self.waker.wake(); + let fut = load.prepare().boxed_local(); + self.preparing_dyn_imports.push(fut); + } +} + #[cfg(test)] pub mod tests { use super::*; - use crate::isolate::js_check; - use crate::isolate::tests::run_in_task; - use crate::isolate::ZeroCopyBuf; + use crate::core_isolate::tests::run_in_task; + use crate::core_isolate::CoreIsolateState; + use crate::js_check; use crate::modules::ModuleSourceFuture; use crate::ops::*; + use crate::ZeroCopyBuf; use std::io; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; #[test] fn test_mods() { - #[derive(Clone, Default)] + #[derive(Default)] struct ModsLoader { pub count: Arc, } @@ -644,9 +721,9 @@ pub mod tests { let mut isolate = EsIsolate::new(loader, StartupData::None, false); - let dispatcher = move |_isolate: &mut CoreIsolate, + let dispatcher = move |_state: &mut CoreIsolateState, control: &[u8], - _zero_copy: Option| + _zero_copy: &mut [ZeroCopyBuf]| -> Op { dispatch_count_.fetch_add(1, Ordering::Relaxed); assert_eq!(control.len(), 1); @@ -685,16 +762,23 @@ pub mod tests { .unwrap(); assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); - let imports = isolate.modules.get_children(mod_a); - assert_eq!( - imports, - Some(&vec![ModuleSpecifier::resolve_url("file:///b.js").unwrap()]) - ); + let state_rc = EsIsolate::state(&isolate); + { + let state = state_rc.borrow(); + let imports = state.modules.get_children(mod_a); + assert_eq!( + imports, + Some(&vec![ModuleSpecifier::resolve_url("file:///b.js").unwrap()]) + ); + } let mod_b = isolate .mod_new(false, "file:///b.js", "export function b() { return 'b' }") .unwrap(); - let imports = isolate.modules.get_children(mod_b).unwrap(); - assert_eq!(imports.len(), 0); + { + let state = state_rc.borrow(); + let imports = state.modules.get_children(mod_b).unwrap(); + assert_eq!(imports.len(), 0); + } js_check(isolate.mod_instantiate(mod_b)); assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); @@ -764,145 +848,57 @@ pub mod tests { }) } - /* - // Note from Bert: I do not understand how this part is supposed to pass. - // For me all these modules load in parallel and, unless I'm missing - // something, that's how it should be. So I disabled the test for now. - #[test] - fn dyn_import_err2() { - #[derive(Clone, Default)] - struct DynImportErr2Loader { - pub count: Arc, - } - - impl ModuleLoader for DynImportErr2Loader { - fn resolve( - &self, - specifier: &str, - referrer: &str, - _is_main: bool, - _is_dyn_import: bool, - ) -> Result { - let c = self.count.fetch_add(1, Ordering::Relaxed); - match c { - 0 => assert_eq!(specifier, "/foo1.js"), - 1 => assert_eq!(specifier, "/foo2.js"), - 2 => assert_eq!(specifier, "/foo3.js"), - _ => unreachable!(), - } - assert_eq!(referrer, "file:///dyn_import_error.js"); - let s = ModuleSpecifier::resolve_import(specifier, referrer).unwrap(); - Ok(s) - } + #[derive(Clone, Default)] + struct DynImportOkLoader { + pub prepare_load_count: Arc, + pub resolve_count: Arc, + pub load_count: Arc, + } - fn load( - &self, - specifier: &ModuleSpecifier, - _maybe_referrer: Option, - ) -> Pin> { - let info = ModuleSource { - module_url_specified: specifier.to_string(), - module_url_found: specifier.to_string(), - code: "# not valid JS".to_owned(), - }; - async move { Ok(info) }.boxed() - } + impl ModuleLoader for DynImportOkLoader { + fn resolve( + &self, + specifier: &str, + referrer: &str, + _is_main: bool, + ) -> Result { + let c = self.resolve_count.fetch_add(1, Ordering::Relaxed); + assert!(c < 4); + assert_eq!(specifier, "./b.js"); + assert_eq!(referrer, "file:///dyn_import3.js"); + let s = ModuleSpecifier::resolve_import(specifier, referrer).unwrap(); + Ok(s) } - // Import multiple modules to demonstrate that after failed dynamic import - // another dynamic import can still be run - run_in_task(|cx| { - let loader = Box::new(DynImportErr2Loader::default()); - let loader1 = loader.clone(); - let mut isolate = EsIsolate::new(loader, StartupData::None, false); - - js_check(isolate.execute( - "file:///dyn_import_error.js", - r#" - (async () => { - await import("/foo1.js"); - })(); - (async () => { - await import("/foo2.js"); - })(); - (async () => { - await import("/foo3.js"); - })(); - "#, - )); + fn load( + &self, + specifier: &ModuleSpecifier, + _maybe_referrer: Option, + _is_dyn_import: bool, + ) -> Pin> { + self.load_count.fetch_add(1, Ordering::Relaxed); + let info = ModuleSource { + module_url_specified: specifier.to_string(), + module_url_found: specifier.to_string(), + code: "export function b() { return 'b' }".to_owned(), + }; + async move { Ok(info) }.boxed() + } - assert_eq!(loader1.count.load(Ordering::Relaxed), 0); - // Now each poll should return error - assert!(match isolate.poll_unpin(cx) { - Poll::Ready(Err(_)) => true, - _ => false, - }); - assert_eq!(loader1.count.load(Ordering::Relaxed), 1); - assert!(match isolate.poll_unpin(cx) { - Poll::Ready(Err(_)) => true, - _ => false, - }); - assert_eq!(loader1.count.load(Ordering::Relaxed), 2); - assert!(match isolate.poll_unpin(cx) { - Poll::Ready(Err(_)) => true, - _ => false, - }); - assert_eq!(loader1.count.load(Ordering::Relaxed), 3); - }) + fn prepare_load( + &self, + _load_id: ModuleLoadId, + _module_specifier: &ModuleSpecifier, + _maybe_referrer: Option, + _is_dyn_import: bool, + ) -> Pin>>> { + self.prepare_load_count.fetch_add(1, Ordering::Relaxed); + async { Ok(()) }.boxed_local() + } } - */ #[test] fn dyn_import_ok() { - #[derive(Clone, Default)] - struct DynImportOkLoader { - pub prepare_load_count: Arc, - pub resolve_count: Arc, - pub load_count: Arc, - } - - impl ModuleLoader for DynImportOkLoader { - fn resolve( - &self, - specifier: &str, - referrer: &str, - _is_main: bool, - ) -> Result { - let c = self.resolve_count.fetch_add(1, Ordering::Relaxed); - assert!(c < 4); - assert_eq!(specifier, "./b.js"); - assert_eq!(referrer, "file:///dyn_import3.js"); - let s = ModuleSpecifier::resolve_import(specifier, referrer).unwrap(); - Ok(s) - } - - fn load( - &self, - specifier: &ModuleSpecifier, - _maybe_referrer: Option, - _is_dyn_import: bool, - ) -> Pin> { - self.load_count.fetch_add(1, Ordering::Relaxed); - let info = ModuleSource { - module_url_specified: specifier.to_string(), - module_url_found: specifier.to_string(), - code: "export function b() { return 'b' }".to_owned(), - }; - async move { Ok(info) }.boxed() - } - - fn prepare_load( - &self, - _load_id: ModuleLoadId, - _module_specifier: &ModuleSpecifier, - _maybe_referrer: Option, - _is_dyn_import: bool, - ) -> Pin>>> { - self.prepare_load_count.fetch_add(1, Ordering::Relaxed); - async { Ok(()) }.boxed_local() - } - } - run_in_task(|cx| { let loader = Rc::new(DynImportOkLoader::default()); let prepare_load_count = loader.prepare_load_count.clone(); @@ -950,4 +946,76 @@ pub mod tests { assert_eq!(load_count.load(Ordering::Relaxed), 2); }) } + + #[test] + fn dyn_import_borrow_mut_error() { + // https://github.com/denoland/deno/issues/6054 + run_in_task(|cx| { + let loader = Rc::new(DynImportOkLoader::default()); + let prepare_load_count = loader.prepare_load_count.clone(); + let mut isolate = EsIsolate::new(loader, StartupData::None, false); + js_check(isolate.execute( + "file:///dyn_import3.js", + r#" + (async () => { + let mod = await import("./b.js"); + if (mod.b() !== 'b') { + throw Error("bad"); + } + // Now do any op + Deno.core.ops(); + })(); + "#, + )); + // First poll runs `prepare_load` hook. + let _ = isolate.poll_unpin(cx); + assert_eq!(prepare_load_count.load(Ordering::Relaxed), 1); + // Second poll triggers error + let _ = isolate.poll_unpin(cx); + }) + } + + #[test] + fn es_snapshot() { + #[derive(Default)] + struct ModsLoader; + + impl ModuleLoader for ModsLoader { + fn resolve( + &self, + specifier: &str, + referrer: &str, + _is_main: bool, + ) -> Result { + assert_eq!(specifier, "file:///main.js"); + assert_eq!(referrer, "."); + let s = ModuleSpecifier::resolve_import(specifier, referrer).unwrap(); + Ok(s) + } + + fn load( + &self, + _module_specifier: &ModuleSpecifier, + _maybe_referrer: Option, + _is_dyn_import: bool, + ) -> Pin> { + unreachable!() + } + } + + let loader = std::rc::Rc::new(ModsLoader::default()); + let mut runtime_isolate = EsIsolate::new(loader, StartupData::None, true); + + let specifier = ModuleSpecifier::resolve_url("file:///main.js").unwrap(); + let source_code = "Deno.core.print('hello\\n')".to_string(); + + let module_id = futures::executor::block_on( + runtime_isolate.load_module(&specifier, Some(source_code)), + ) + .unwrap(); + + js_check(runtime_isolate.mod_evaluate(module_id)); + + let _snapshot = runtime_isolate.snapshot(); + } } diff --git a/core/examples/http_bench.js b/core/examples/http_bench.js index d9878cbe741d35..a893dab40816df 100644 --- a/core/examples/http_bench.js +++ b/core/examples/http_bench.js @@ -36,18 +36,18 @@ const scratchBytes = new Uint8Array( ); assert(scratchBytes.byteLength === 3 * 4); -function send(promiseId, opId, rid, zeroCopy = null) { +function send(promiseId, opId, rid, ...zeroCopy) { scratch32[0] = promiseId; scratch32[1] = rid; scratch32[2] = -1; - return Deno.core.dispatch(opId, scratchBytes, zeroCopy); + return Deno.core.dispatch(opId, scratchBytes, ...zeroCopy); } /** Returns Promise */ -function sendAsync(opId, rid, zeroCopy = null) { +function sendAsync(opId, rid, ...zeroCopy) { const promiseId = nextPromiseId++; const p = createResolvable(); - const buf = send(promiseId, opId, rid, zeroCopy); + const buf = send(promiseId, opId, rid, ...zeroCopy); if (buf) { const record = recordFromBuf(buf); // Sync result. diff --git a/core/examples/http_bench.rs b/core/examples/http_bench.rs index f728b2a69d3c66..233864fac1e6e7 100644 --- a/core/examples/http_bench.rs +++ b/core/examples/http_bench.rs @@ -4,6 +4,7 @@ extern crate derive_deref; extern crate log; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::Op; use deno_core::ResourceTable; use deno_core::Script; @@ -77,7 +78,7 @@ impl From for RecordBuf { } struct Isolate { - core_isolate: Box, // Unclear why CoreIsolate::new() returns a box. + core_isolate: CoreIsolate, state: State, } @@ -112,19 +113,19 @@ impl Isolate { fn register_sync_op(&mut self, name: &'static str, handler: F) where - F: 'static + Fn(State, u32, Option) -> Result, + F: 'static + Fn(State, u32, &mut [ZeroCopyBuf]) -> Result, { let state = self.state.clone(); - let core_handler = move |_isolate: &mut CoreIsolate, + let core_handler = move |_isolate_state: &mut CoreIsolateState, control_buf: &[u8], - zero_copy_buf: Option| + zero_copy_bufs: &mut [ZeroCopyBuf]| -> Op { let state = state.clone(); let record = Record::from(control_buf); let is_sync = record.promise_id == 0; assert!(is_sync); - let result: i32 = match handler(state, record.rid, zero_copy_buf) { + let result: i32 = match handler(state, record.rid, zero_copy_bufs) { Ok(r) => r as i32, Err(_) => -1, }; @@ -138,24 +139,25 @@ impl Isolate { fn register_op( &mut self, name: &'static str, - handler: impl Fn(State, u32, Option) -> F + Copy + 'static, + handler: impl Fn(State, u32, &mut [ZeroCopyBuf]) -> F + Copy + 'static, ) where F: TryFuture, F::Ok: TryInto, >::Error: Debug, { let state = self.state.clone(); - let core_handler = move |_isolate: &mut CoreIsolate, + let core_handler = move |_isolate_state: &mut CoreIsolateState, control_buf: &[u8], - zero_copy_buf: Option| + zero_copy_bufs: &mut [ZeroCopyBuf]| -> Op { let state = state.clone(); let record = Record::from(control_buf); let is_sync = record.promise_id == 0; assert!(!is_sync); + let mut zero_copy = zero_copy_bufs.to_vec(); let fut = async move { - let op = handler(state, record.rid, zero_copy_buf); + let op = handler(state, record.rid, &mut zero_copy); let result = op .map_ok(|r| r.try_into().expect("op result does not fit in i32")) .unwrap_or_else(|_| -1) @@ -181,7 +183,7 @@ impl Future for Isolate { fn op_close( state: State, rid: u32, - _buf: Option, + _buf: &mut [ZeroCopyBuf], ) -> Result { debug!("close rid={}", rid); let resource_table = &mut state.borrow_mut().resource_table; @@ -194,7 +196,7 @@ fn op_close( fn op_listen( state: State, _rid: u32, - _buf: Option, + _buf: &mut [ZeroCopyBuf], ) -> Result { debug!("listen"); let addr = "127.0.0.1:4544".parse::().unwrap(); @@ -208,7 +210,7 @@ fn op_listen( fn op_accept( state: State, rid: u32, - _buf: Option, + _buf: &mut [ZeroCopyBuf], ) -> impl TryFuture { debug!("accept rid={}", rid); @@ -226,9 +228,11 @@ fn op_accept( fn op_read( state: State, rid: u32, - buf: Option, + bufs: &mut [ZeroCopyBuf], ) -> impl TryFuture { - let mut buf = buf.unwrap(); + assert_eq!(bufs.len(), 1, "Invalid number of arguments"); + let mut buf = bufs[0].clone(); + debug!("read rid={}", rid); poll_fn(move |cx| { @@ -243,9 +247,10 @@ fn op_read( fn op_write( state: State, rid: u32, - buf: Option, + bufs: &mut [ZeroCopyBuf], ) -> impl TryFuture { - let buf = buf.unwrap(); + assert_eq!(bufs.len(), 1, "Invalid number of arguments"); + let buf = bufs[0].clone(); debug!("write rid={}", rid); poll_fn(move |cx| { diff --git a/core/lib.rs b/core/lib.rs index ffccc8febb5bc4..47ded645e71f76 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -8,30 +8,46 @@ extern crate lazy_static; #[macro_use] extern crate log; -mod any_error; mod bindings; +mod core_isolate; +mod errors; mod es_isolate; mod flags; -mod isolate; -mod js_errors; mod module_specifier; mod modules; mod ops; pub mod plugin_api; mod resources; mod shared_queue; +mod zero_copy_buf; pub use rusty_v8 as v8; -pub use crate::any_error::*; -pub use crate::es_isolate::*; +pub use crate::core_isolate::js_check; +pub use crate::core_isolate::CoreIsolate; +pub use crate::core_isolate::CoreIsolateState; +pub use crate::core_isolate::Script; +pub use crate::core_isolate::Snapshot; +pub use crate::core_isolate::StartupData; +pub use crate::errors::ErrBox; +pub use crate::errors::JSError; +pub use crate::es_isolate::EsIsolate; +pub use crate::es_isolate::EsIsolateState; pub use crate::flags::v8_set_flags; -pub use crate::isolate::*; -pub use crate::js_errors::*; -pub use crate::module_specifier::*; -pub use crate::modules::*; -pub use crate::ops::*; -pub use crate::resources::*; +pub use crate::module_specifier::ModuleResolutionError; +pub use crate::module_specifier::ModuleSpecifier; +pub use crate::modules::ModuleId; +pub use crate::modules::ModuleLoadId; +pub use crate::modules::ModuleLoader; +pub use crate::modules::ModuleSource; +pub use crate::modules::ModuleSourceFuture; +pub use crate::modules::RecursiveModuleLoad; +pub use crate::ops::Buf; +pub use crate::ops::Op; +pub use crate::ops::OpAsyncFuture; +pub use crate::ops::OpId; +pub use crate::resources::ResourceTable; +pub use crate::zero_copy_buf::ZeroCopyBuf; pub fn v8_version() -> &'static str { v8::V8::get_version() diff --git a/core/modules.rs b/core/modules.rs index 632df2dd000b2f..ca850d0bb04f65 100644 --- a/core/modules.rs +++ b/core/modules.rs @@ -2,10 +2,8 @@ use rusty_v8 as v8; -use crate::any_error::ErrBox; -use crate::es_isolate::ModuleId; -use crate::es_isolate::ModuleLoadId; use crate::module_specifier::ModuleSpecifier; +use crate::ErrBox; use futures::future::FutureExt; use futures::stream::FuturesUnordered; use futures::stream::Stream; @@ -25,6 +23,9 @@ lazy_static! { pub static ref NEXT_LOAD_ID: AtomicI32 = AtomicI32::new(0); } +pub type ModuleId = i32; +pub type ModuleLoadId = i32; + /// EsModule source code that will be loaded into V8. /// /// Users can implement `Into` for different file types that @@ -548,7 +549,8 @@ macro_rules! include_crate_modules { mod tests { use super::*; use crate::es_isolate::EsIsolate; - use crate::isolate::js_check; + use crate::js_check; + use crate::StartupData; use futures::future::FutureExt; use std::error::Error; use std::fmt; @@ -556,6 +558,12 @@ mod tests { use std::sync::Arc; use std::sync::Mutex; + // TODO(ry) Sadly FuturesUnordered requires the current task to be set. So + // even though we are only using poll() in these tests and not Tokio, we must + // nevertheless run it in the tokio executor. Ideally run_in_task can be + // removed in the future. + use crate::core_isolate::tests::run_in_task; + struct MockLoader { pub loads: Arc>>, } @@ -716,13 +724,6 @@ mod tests { if (import.meta.url != 'file:///d.js') throw Error(); "#; - // TODO(ry) Sadly FuturesUnordered requires the current task to be set. So - // even though we are only using poll() in these tests and not Tokio, we must - // nevertheless run it in the tokio executor. Ideally run_in_task can be - // removed in the future. - use crate::isolate::tests::run_in_task; - use crate::isolate::StartupData; - #[test] fn test_recursive_load() { let loader = MockLoader::new(); @@ -744,7 +745,9 @@ mod tests { ] ); - let modules = &isolate.modules; + let state_rc = EsIsolate::state(&isolate); + let state = state_rc.borrow(); + let modules = &state.modules; assert_eq!(modules.get_id("file:///a.js"), Some(a_id)); let b_id = modules.get_id("file:///b.js").unwrap(); let c_id = modules.get_id("file:///c.js").unwrap(); @@ -806,7 +809,9 @@ mod tests { ] ); - let modules = &isolate.modules; + let state_rc = EsIsolate::state(&isolate); + let state = state_rc.borrow(); + let modules = &state.modules; assert_eq!(modules.get_id("file:///circular1.js"), Some(circular1_id)); let circular2_id = modules.get_id("file:///circular2.js").unwrap(); @@ -877,7 +882,9 @@ mod tests { ] ); - let modules = &isolate.modules; + let state_rc = EsIsolate::state(&isolate); + let state = state_rc.borrow(); + let modules = &state.modules; assert_eq!(modules.get_id("file:///redirect1.js"), Some(redirect1_id)); @@ -1015,7 +1022,9 @@ mod tests { vec!["file:///b.js", "file:///c.js", "file:///d.js"] ); - let modules = &isolate.modules; + let state_rc = EsIsolate::state(&isolate); + let state = state_rc.borrow(); + let modules = &state.modules; assert_eq!(modules.get_id("file:///main_with_code.js"), Some(main_id)); let b_id = modules.get_id("file:///b.js").unwrap(); diff --git a/core/ops.rs b/core/ops.rs index 0361f5ee9f1bce..bd9d58283787ab 100644 --- a/core/ops.rs +++ b/core/ops.rs @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -use crate::CoreIsolate; +use crate::core_isolate::CoreIsolateState; use crate::ZeroCopyBuf; use futures::Future; use std::collections::HashMap; @@ -22,7 +22,7 @@ pub enum Op { /// Main type describing op pub type OpDispatcher = - dyn Fn(&mut CoreIsolate, &[u8], Option) -> Op + 'static; + dyn Fn(&mut CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op + 'static; #[derive(Default)] pub struct OpRegistry { @@ -33,8 +33,8 @@ pub struct OpRegistry { impl OpRegistry { pub fn new() -> Self { let mut registry = Self::default(); - let op_id = registry.register("ops", |isolate, _, _| { - let buf = isolate.op_registry.json_map(); + let op_id = registry.register("ops", |state, _, _| { + let buf = state.op_registry.json_map(); Op::Sync(buf) }); assert_eq!(op_id, 0); @@ -43,7 +43,7 @@ impl OpRegistry { pub fn register(&mut self, name: &str, op: F) -> OpId where - F: Fn(&mut CoreIsolate, &[u8], Option) -> Op + 'static, + F: Fn(&mut CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op + 'static, { let op_id = self.dispatchers.len() as u32; @@ -68,6 +68,7 @@ impl OpRegistry { #[test] fn test_op_registry() { + use crate::CoreIsolate; use std::sync::atomic; use std::sync::Arc; let mut op_registry = OpRegistry::new(); @@ -86,10 +87,12 @@ fn test_op_registry() { expected.insert("test".to_string(), 1); assert_eq!(op_registry.name_to_id, expected); - let mut isolate = CoreIsolate::new(crate::StartupData::None, false); + let isolate = CoreIsolate::new(crate::StartupData::None, false); let dispatch = op_registry.get(test_id).unwrap(); - let res = dispatch(&mut isolate, &[], None); + let state_rc = CoreIsolate::state(&isolate); + let mut state = state_rc.borrow_mut(); + let res = dispatch(&mut state, &[], &mut []); if let Op::Sync(buf) = res { assert_eq!(buf.len(), 0); } else { @@ -102,6 +105,7 @@ fn test_op_registry() { #[test] fn register_op_during_call() { + use crate::CoreIsolate; use std::sync::atomic; use std::sync::Arc; use std::sync::Mutex; @@ -126,13 +130,17 @@ fn register_op_during_call() { }; assert!(test_id != 0); - let mut isolate = CoreIsolate::new(crate::StartupData::None, false); + let isolate = CoreIsolate::new(crate::StartupData::None, false); let dispatcher1 = { let g = op_registry.lock().unwrap(); g.get(test_id).unwrap() }; - dispatcher1(&mut isolate, &[], None); + { + let state_rc = CoreIsolate::state(&isolate); + let mut state = state_rc.borrow_mut(); + dispatcher1(&mut state, &[], &mut []); + } let mut expected = HashMap::new(); expected.insert("ops".to_string(), 0); @@ -147,7 +155,9 @@ fn register_op_during_call() { let g = op_registry.lock().unwrap(); g.get(2).unwrap() }; - let res = dispatcher2(&mut isolate, &[], None); + let state_rc = CoreIsolate::state(&isolate); + let mut state = state_rc.borrow_mut(); + let res = dispatcher2(&mut state, &[], &mut []); if let Op::Sync(buf) = res { assert_eq!(buf.len(), 0); } else { diff --git a/core/plugin_api.rs b/core/plugin_api.rs index 2e93fdb77e5eec..16f5d4a365b6c4 100644 --- a/core/plugin_api.rs +++ b/core/plugin_api.rs @@ -15,8 +15,7 @@ pub use crate::ZeroCopyBuf; pub type InitFn = fn(&mut dyn Interface); -pub type DispatchOpFn = - fn(&mut dyn Interface, &[u8], Option) -> Op; +pub type DispatchOpFn = fn(&mut dyn Interface, &[u8], &mut [ZeroCopyBuf]) -> Op; pub trait Interface { fn register_op(&mut self, name: &str, dispatcher: DispatchOpFn) -> OpId; diff --git a/core/zero_copy_buf.rs b/core/zero_copy_buf.rs new file mode 100644 index 00000000000000..be61d5f9841892 --- /dev/null +++ b/core/zero_copy_buf.rs @@ -0,0 +1,71 @@ +use crate::bindings; +use rusty_v8 as v8; +use std::ops::Deref; +use std::ops::DerefMut; + +/// A ZeroCopyBuf encapsulates a slice that's been borrowed from a JavaScript +/// ArrayBuffer object. JavaScript objects can normally be garbage collected, +/// but the existence of a ZeroCopyBuf inhibits this until it is dropped. It +/// behaves much like an Arc<[u8]>, although a ZeroCopyBuf currently can't be +/// cloned. +#[derive(Clone)] +pub struct ZeroCopyBuf { + backing_store: v8::SharedRef, + byte_offset: usize, + byte_length: usize, +} + +unsafe impl Send for ZeroCopyBuf {} + +impl ZeroCopyBuf { + pub fn new<'s>( + scope: &mut impl v8::ToLocal<'s>, + view: v8::Local, + ) -> Self { + let backing_store = view.buffer(scope).unwrap().get_backing_store(); + let byte_offset = view.byte_offset(); + let byte_length = view.byte_length(); + Self { + backing_store, + byte_offset, + byte_length, + } + } +} + +impl Deref for ZeroCopyBuf { + type Target = [u8]; + fn deref(&self) -> &[u8] { + unsafe { + bindings::get_backing_store_slice( + &self.backing_store, + self.byte_offset, + self.byte_length, + ) + } + } +} + +impl DerefMut for ZeroCopyBuf { + fn deref_mut(&mut self) -> &mut [u8] { + unsafe { + bindings::get_backing_store_slice_mut( + &self.backing_store, + self.byte_offset, + self.byte_length, + ) + } + } +} + +impl AsRef<[u8]> for ZeroCopyBuf { + fn as_ref(&self) -> &[u8] { + &*self + } +} + +impl AsMut<[u8]> for ZeroCopyBuf { + fn as_mut(&mut self) -> &mut [u8] { + &mut *self + } +} diff --git a/deno_typescript/Cargo.toml b/deno_typescript/Cargo.toml index 66a278576fe1e0..50802ce423160c 100644 --- a/deno_typescript/Cargo.toml +++ b/deno_typescript/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deno_typescript" -version = "0.45.2" +version = "0.47.1" license = "MIT" description = "To compile TypeScript to a snapshot during build.rs" repository = "https://github.com/denoland/deno" @@ -19,6 +19,6 @@ exclude = [ path = "lib.rs" [dependencies] -deno_core = { path = "../core", version = "0.45.0" } -serde_json = "1.0.52" -serde = { version = "1.0.106", features = ["derive"] } +deno_core = { path = "../core", version = "0.47.1" } +serde_json = "1.0.53" +serde = { version = "1.0.111", features = ["derive"] } diff --git a/deno_typescript/compiler_main.js b/deno_typescript/compiler_main.js index 31f539a2753cb9..847f3435f65f2a 100644 --- a/deno_typescript/compiler_main.js +++ b/deno_typescript/compiler_main.js @@ -5,6 +5,8 @@ // understood by the TypeScript language service, so it allows type safety // checking in VSCode. +"use strict"; + const ASSETS = "$asset$"; /** diff --git a/deno_typescript/lib.rs b/deno_typescript/lib.rs index 590e2991713fd6..52fb00b767c8b6 100644 --- a/deno_typescript/lib.rs +++ b/deno_typescript/lib.rs @@ -9,6 +9,7 @@ mod ops; use deno_core::js_check; pub use deno_core::v8_set_flags; use deno_core::CoreIsolate; +use deno_core::CoreIsolateState; use deno_core::ErrBox; use deno_core::ModuleSpecifier; use deno_core::Op; @@ -49,22 +50,22 @@ pub struct TSState { fn compiler_op( ts_state: Arc>, dispatcher: D, -) -> impl Fn(&mut CoreIsolate, &[u8], Option) -> Op +) -> impl Fn(&mut CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op where D: Fn(&mut TSState, &[u8]) -> Op, { - move |_isolate: &mut CoreIsolate, + move |_state: &mut CoreIsolateState, control: &[u8], - zero_copy_buf: Option| + zero_copy_bufs: &mut [ZeroCopyBuf]| -> Op { - assert!(zero_copy_buf.is_none()); // zero_copy_buf unused in compiler. + assert!(zero_copy_bufs.is_empty()); // zero_copy_bufs unused in compiler. let mut s = ts_state.lock().unwrap(); dispatcher(&mut s, control) } } pub struct TSIsolate { - isolate: Box, + isolate: CoreIsolate, state: Arc>, } @@ -331,15 +332,15 @@ pub fn trace_serializer() { /// CoreIsolate. pub fn op_fetch_asset( custom_assets: HashMap, -) -> impl Fn(&mut CoreIsolate, &[u8], Option) -> Op { +) -> impl Fn(&mut CoreIsolateState, &[u8], &mut [ZeroCopyBuf]) -> Op { for (_, path) in custom_assets.iter() { println!("cargo:rerun-if-changed={}", path.display()); } - move |_isolate: &mut CoreIsolate, + move |_state: &mut CoreIsolateState, control: &[u8], - zero_copy_buf: Option| + zero_copy_bufs: &mut [ZeroCopyBuf]| -> Op { - assert!(zero_copy_buf.is_none()); // zero_copy_buf unused in this op. + assert!(zero_copy_bufs.is_empty()); // zero_copy_bufs unused in this op. let name = std::str::from_utf8(control).unwrap(); let asset_code = if let Some(source_code) = get_asset(name) { diff --git a/deno_typescript/system_loader.js b/deno_typescript/system_loader.js index 0004d055d62cb0..fdf1fa87230f41 100644 --- a/deno_typescript/system_loader.js +++ b/deno_typescript/system_loader.js @@ -2,6 +2,8 @@ // This is a specialised implementation of a System module loader. +"use strict"; + // @ts-nocheck /* eslint-disable */ let System, __instantiateAsync, __instantiate; diff --git a/docs/contributing.md b/docs/contributing.md index b48d18450771e6..e4003e5df3715b 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -9,9 +9,9 @@ - If you are going to work on an issue, mention so in the issue comments _before_ you start working on the issue. -- Please be professional in the forums. Don't know what professional means? - [Read Rust's code of conduct](https://www.rust-lang.org/policies/code-of-conduct). - Have a problem? Email ry@tinyclouds.org. +- Please be professional in the forums. We follow + [Rust's code of conduct](https://www.rust-lang.org/policies/code-of-conduct) + (CoC) Have a problem? Email ry@tinyclouds.org. ## Development @@ -45,6 +45,11 @@ Please list how this functionality is done in Go, Node, Rust, and Python. As an example, see how `Deno.rename()` was proposed and added in [PR #671](https://github.com/denoland/deno/pull/671). +## Releases + +Summary of the changes from previous releases can be found +[here](https://github.com/denoland/deno/releases). + ## Documenting APIs It is important to document public APIs and we want to do that inline with the diff --git a/docs/contributing/style_guide.md b/docs/contributing/style_guide.md index ed23f31e13b8e1..0bd7c628f34bb7 100644 --- a/docs/contributing/style_guide.md +++ b/docs/contributing/style_guide.md @@ -314,3 +314,16 @@ export function foo(): string { `https://deno.land/std/` is intended to be baseline functionality that all Deno programs can rely on. We want to guarantee to users that this code does not include potentially unreviewed third party code. + +#### Document and maintain browser compatiblity. + +If a module is browser compatible, include the following in the JSDoc at the top +of the module: + +```ts +/** This module is browser compatible. */ +``` + +Maintain browser compatibility for such a module by either not using the global +`Deno` namespace or feature-testing for it. Make sure any new dependencies are +also browser compatible. diff --git a/docs/examples/tcp_echo.md b/docs/examples/tcp_echo.md index 360c5faccd58ea..5dfc71b8317bba 100644 --- a/docs/examples/tcp_echo.md +++ b/docs/examples/tcp_echo.md @@ -1,7 +1,7 @@ ## TCP echo server -This is an example of a simple server which accepts connections on port 8080, -and returns to the client anything it sends. +This is an example of a server which accepts connections on port 8080, and +returns to the client anything it sends. ```ts const listener = Deno.listen({ port: 8080 }); diff --git a/docs/getting_started.md b/docs/getting_started.md index 159d3de6486d1f..d36623c1875a19 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -3,7 +3,10 @@ In this chapter we'll discuss: - Installing Deno -- Running a simple `Hello World` script +- Setting up your environment +- Running a `Hello World` script - Writing our own script +- Command line interface +- Understanding permissions - Using Deno with TypeScript - Using WebAssembly diff --git a/docs/getting_started/command_line_interface.md b/docs/getting_started/command_line_interface.md new file mode 100644 index 00000000000000..e8538cd3f8afd5 --- /dev/null +++ b/docs/getting_started/command_line_interface.md @@ -0,0 +1,124 @@ +## Command line interface + +Deno is a command line program. You should be familiar with some simple commands +having followed the examples thus far and already understand the basics of shell +usage. + +There are multiple ways of viewing the main help text: + +```shell +# Using the subcommand. +deno help + +# Using the short flag -- outputs the same as above. +deno -h + +# Using the long flag -- outputs more detailed help text where available. +deno --help +``` + +Deno's CLI is subcommand-based. The above commands should show you a list of +those supported, such as `deno bundle`. To see subcommand-specific help for +`bundle`, you can similarly run one of: + +```shell +deno help bundle +deno bundle -h +deno bundle --help +``` + +Detailed guides to each subcommand can be found [here](../tools.md). + +### Script arguments + +Separately from the Deno runtime flags, you can pass user-space arguments to the +script you are running by specifying them after the script name: + +```shell +deno run main.ts a b -c --quiet +``` + +```ts +// main.ts +console.log(Deno.args); // [ "a", "b", "-c", "--quiet" ] +``` + +**Note that anything passed after the script name will be passed as a script +argument and not consumed as a Deno runtime flag.** This leads to the following +pitfall: + +```shell +# Good. We grant net permission to net_client.ts. +deno run --allow-net net_client.ts + +# Bad! --allow-net was passed to Deno.args, throws a net permission error. +deno run net_client.ts --allow-net +``` + +Some see it as unconventional that: + +> a non-positional flag is parsed differently depending on its position. + +However: + +1. This is the most logical way of distinguishing between runtime flags and + script arguments. +2. This is the most ergonomic way of distinguishing between runtime flags and + script arguments. +3. This is, in fact, the same behaviour as that of any other popular runtime. + - Try `node -c index.js` and `node index.js -c`. The first will only do a + syntax check on `index.js` as per Node's `-c` flag. The second will + _execute_ `index.js` with `-c` passed to `require("process").argv`. + +--- + +There exist logical groups of flags that are shared between related subcommands. +We discuss these below. + +### Integrity flags + +Affect commands which can download resources to the cache: `deno cache`, +`deno run` and `deno test`. + +``` +--lock Check the specified lock file +--lock-write Write lock file. Use with --lock. +``` + +Find out more about these +[here](../linking_to_external_code/integrity_checking.md). + +### Cache and compilation flags + +Affect commands which can populate the cache: `deno cache`, `deno run` and +`deno test`. As well as the flags above this includes those which affect module +resolution, compilation configuration etc. + +``` +--config Load tsconfig.json configuration file +--importmap UNSTABLE: Load import map file +--no-remote Do not resolve remote modules +--reload= Reload source code cache (recompile TypeScript) +--unstable Enable unstable APIs +``` + +### Runtime flags + +Affect commands which execute user code: `deno run` and `deno test`. These +include all of the above as well as the following. + +#### Permission flags + +These are listed [here](./permissions.md#permissions-list). + +#### Other runtime flags + +More flags which affect the execution environment. + +``` +--cached-only Require that remote dependencies are already cached +--inspect= activate inspector on host:port ... +--inspect-brk= activate inspector on host:port and break at ... +--seed Seed Math.random() +--v8-flags= Set V8 command line options. For help: ... +``` diff --git a/docs/getting_started/first_steps.md b/docs/getting_started/first_steps.md index 4b3dd51086a990..4a551c41566f1e 100644 --- a/docs/getting_started/first_steps.md +++ b/docs/getting_started/first_steps.md @@ -1,7 +1,6 @@ ## First steps -This page contains some simple examples to teach you about the fundamentals of -Deno. +This page contains some examples to teach you about the fundamentals of Deno. This document assumes that you have some prior knowledge of JavaScript, especially about `async`/`await`. If you have no prior knowledge of JavaScript, @@ -14,8 +13,8 @@ before attempting to start with Deno. Deno is a runtime for JavaScript/TypeScript which tries to be web compatible and use modern features wherever possible. -Browser compatibility means, a simple `Hello World` program in Deno is the same -as the one you can run in the browser: +Browser compatibility means a `Hello World` program in Deno is the same as the +one you can run in the browser: ```ts console.log("Welcome to Deno 🦕"); @@ -87,9 +86,9 @@ In this program each command-line argument is assumed to be a filename, the file is opened, and printed to stdout. ```ts -for (let i = 0; i < Deno.args.length; i++) { - let filename = Deno.args[i]; - let file = await Deno.open(filename); +const filenames = Deno.args; +for (const filename of filenames) { + const file = await Deno.open(filename); await Deno.copy(file, Deno.stdout); file.close(); } @@ -106,14 +105,16 @@ Try the program: deno run --allow-read https://deno.land/std/examples/cat.ts /etc/passwd ``` -### A simple TCP server +### TCP server -This is an example of a simple server which accepts connections on port 8080, -and returns to the client anything it sends. +This is an example of a server which accepts connections on port 8080, and +returns to the client anything it sends. ```ts -const listener = Deno.listen({ port: 8080 }); -console.log("listening on 0.0.0.0:8080"); +const hostname = "0.0.0.0"; +const port = 8080; +const listener = Deno.listen({ hostname, port }); +console.log(`Listening on ${hostname}:${port}`); for await (const conn of listener) { Deno.copy(conn, conn); } diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index 158e18133f20dd..8d665d88b1d9bf 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -54,16 +54,27 @@ executable bit on macOS and Linux. To test your installation, run `deno --version`. If this prints the Deno version to the console the installation was successful. -Use `deno help` to see help text documenting Deno's flags and usage. Use -`deno help ` for subcommand-specific flags. +Use `deno help` to see help text documenting Deno's flags and usage. Get a +detailed guide on the CLI [here](./command_line_interface.md). ### Updating -To update a previously installed version of Deno, you can run `deno upgrade`. +To update a previously installed version of Deno, you can run: + +```shell +deno upgrade +``` + This will fetch the latest release from [github.com/denoland/deno/releases](https://github.com/denoland/deno/releases), unzip it, and replace your current executable with it. +You can also use this utility to install a specific version of Deno: + +```shell +deno upgrade --version 1.0.1 +``` + ### Building from source Information about how to build from source can be found in the `Contributing` diff --git a/docs/getting_started/permissions.md b/docs/getting_started/permissions.md index c5880225aca633..317d850463664d 100644 --- a/docs/getting_started/permissions.md +++ b/docs/getting_started/permissions.md @@ -1,15 +1,50 @@ ## Permissions - +Deno is secure by default. Therefore, unless you specifically enable it, a deno +module has no file, network, or environment access for example. Access to +security sensitive areas or functions requires the use of permissions to be +granted to a deno process on the command line. - +For the following example, `mod.ts` has been granted read-only access to the +file system. It cannot write to it, or perform any other security sensitive +functions. + +```shell +deno run --allow-read mod.ts +``` + +### Permissions list + +The following permissions are available: + +- **-A, --allow-all** Allow all permissions. This disables all security. +- **--allow-env** Allow environment access for things like getting and setting + of environment variables. +- **--allow-hrtime** Allow high resolution time measurement. High resolution + time can be used in timing attacks and fingerprinting. +- **--allow-net=\** Allow network access. You can specify an + optional, comma separated list of domains to provide a whitelist of allowed + domains. +- **--allow-plugin** Allow loading plugins. Please note that --allow-plugin is + an unstable feature. +- **--allow-read=\** Allow file system read access. You can specify + an optional, comma separated list of directories or files to provide a + whitelist of allowed file system access. +- **--allow-run** Allow running subprocesses. Be aware that subprocesses are not + run in a sandbox and therefore do not have the same security restrictions as + the deno process. Therefore, use with caution. +- **--allow-write=\** Allow file system write access. You can + specify an optional, comma separated list of directories or files to provide a + whitelist of allowed file system access. ### Permissions whitelist -Deno also allows you to control the granularity of permissions with whitelists. +Deno also allows you to control the granularity of some permissions with +whitelists. This example restricts file system access by whitelisting only the `/usr` -directory: +directory, however the execution fails as the process was attempting to access a +file in the `/etc` directory: ```shell $ deno run --allow-read=/usr https://deno.land/std/examples/cat.ts /etc/passwd @@ -41,6 +76,9 @@ This is an example on how to whitelist hosts/urls: $ deno run --allow-net=github.com,deno.land fetch.ts ``` +If `fetch.ts` tries to establish network connections to any other domain, the +process will fail. + Allow net calls to any host/url: ```shell diff --git a/docs/getting_started/setup_your_environment.md b/docs/getting_started/setup_your_environment.md index a0d04b3850b6bc..3c115354ad49dc 100644 --- a/docs/getting_started/setup_your_environment.md +++ b/docs/getting_started/setup_your_environment.md @@ -8,8 +8,8 @@ IDE of choice. There are several env vars that control how Deno behaves: -`DENO_DIR` defaults to `$HOME/.deno` but can be set to any path to control where -generated and cached source code is written and read to. +`DENO_DIR` defaults to `$HOME/.cache/deno` but can be set to any path to control +where generated and cached source code is written and read to. `NO_COLOR` will turn off color output if set. See https://no-color.org/. User code can test if `NO_COLOR` was set without having `--allow-env` by using the @@ -49,25 +49,59 @@ The community has developed extensions for some editors to solve these issues: The beta version of [vscode_deno](https://github.com/denoland/vscode_deno) is published on the -[Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=justjavac.vscode-deno). +[Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=denoland.vscode-deno). Please report any issues. #### JetBrains IDEs -Support for JetBrains IDEs is not yet available, but you can follow and upvote -these issues to stay up to date: +Support for JetBrains IDEs is available through +[the Deno plugin](https://plugins.jetbrains.com/plugin/14382-deno). -- https://youtrack.jetbrains.com/issue/WEB-41607 -- https://youtrack.jetbrains.com/issue/WEB-42983 -- https://youtrack.jetbrains.com/issue/WEB-31667 +For more information on how to set-up your JetBrains IDE for Deno, read +[this comment](https://youtrack.jetbrains.com/issue/WEB-41607#focus=streamItem-27-4160152.0-0) +on YouTrack. #### Vim and NeoVim Vim works fairly well for Deno/TypeScript if you install [CoC](https://github.com/neoclide/coc.nvim) (intellisense engine and language -server protocol). After CoC itself is installed, from inside Vim, simply run -`:CocInstall coc-deno`. From now on, things like `gd` (go to definition) and -`gr` (goto/find references) should work. +server protocol). + +After CoC is installed, from inside Vim, run`:CocInstall coc-tsserver` and +`:CocInstall coc-deno`. To get autocompletion working for Deno type definitions +run `:CocCommand deno.types`. Optionally restart the CoC server `:CocRestart`. +From now on, things like `gd` (go to definition) and `gr` (goto/find references) +should work. + +#### Emacs + +Emacs works pretty well for a TypeScript project targeted to Deno by using a +combination of [tide](https://github.com/ananthakumaran/tide) which is the +canonical way of using TypeScript within Emacs and +[typescript-deno-plugin](https://github.com/justjavac/typescript-deno-plugin) +which is what is used by the +[official VSCode extension for Deno](https://github.com/denoland/vscode_deno). + +To use it, first make sure that `tide` is setup for your instance of Emacs. +Next, as instructed on the +[typescript-deno-plugin](https://github.com/justjavac/typescript-deno-plugin) +page, first `npm install --save-dev typescript-deno-plugin typescript` in your +project (`npm init -y` as necessary), then add the following block to your +`tsconfig.json` and you are off to the races! + +```json +{ + "compilerOptions": { + "plugins": [ + { + "name": "typescript-deno-plugin", + "enable": true, // default is `true` + "importmap": "import_map.json" + } + ] + } +} +``` If you don't see your favorite IDE on this list, maybe you can develop an extension. Our [community Discord group](https://discord.gg/TGMHGv6) can give diff --git a/docs/getting_started/typescript.md b/docs/getting_started/typescript.md index 197c41482e2007..7689273db15437 100644 --- a/docs/getting_started/typescript.md +++ b/docs/getting_started/typescript.md @@ -2,12 +2,19 @@ -### Using external type definitions - Deno supports both JavaScript and TypeScript as first class languages at runtime. This means it requires fully qualified module names, including the extension (or a server providing the correct media type). In addition, Deno has -no "magical" module resolution. +no "magical" module resolution. Instead, imported modules are specified as files +(including extensions) or fully qualified URL imports. Typescript modules can be +directly imported. E.g. + +``` +import { Response } from "https://deno.land/std@0.53.0/http/server.ts"; +import { queue } from "./collections.ts"; +``` + +### Using external type definitions The out of the box TypeScript compiler though relies on both extension-less modules and the Node.js module resolution logic to apply types to JavaScript @@ -98,7 +105,7 @@ way to support customization a configuration file such as `tsconfig.json` might be provided to Deno on program execution. You need to explicitly tell Deno where to look for this configuration by setting -the `-c` argument when executing your application. +the `-c` (or `--config`) argument when executing your application. ```shell deno run -c tsconfig.json mod.ts diff --git a/docs/runtime.md b/docs/runtime.md index 735ff328e99154..50cca6def8adb6 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -9,8 +9,11 @@ on For APIs where a web standard already exists, like `fetch` for HTTP requests, Deno uses these rather than inventing a new proprietary API. -The documentation for all of these Web APIs can be found on +The detailed documentation for implemented Web APIs can be found on [doc.deno.land](https://doc.deno.land/https/raw.githubusercontent.com/denoland/deno/master/cli/js/lib.deno.shared_globals.d.ts). +Additionally, a full list of the Web APIs which Deno implements is also +available +[in the repository](https://github.com/denoland/deno/blob/master/cli/js/web/README.md). The TypeScript definitions for the implemented web APIs can be found in the [`lib.deno.shared_globals.d.ts`](https://github.com/denoland/deno/blob/master/cli/js/lib.deno.shared_globals.d.ts) diff --git a/docs/runtime/compiler_apis.md b/docs/runtime/compiler_apis.md index ebb14f6b3cb208..c2d2348360dde5 100644 --- a/docs/runtime/compiler_apis.md +++ b/docs/runtime/compiler_apis.md @@ -52,7 +52,7 @@ const [diagnostics, emitMap] = await Deno.compile( ); ``` -In this case `emitMap` will contain a simple `console.log()` statement. +In this case `emitMap` will contain a `console.log()` statement. ### `Deno.bundle()` diff --git a/docs/runtime/program_lifecycle.md b/docs/runtime/program_lifecycle.md index 4a94d6724bbada..7f60350485facf 100644 --- a/docs/runtime/program_lifecycle.md +++ b/docs/runtime/program_lifecycle.md @@ -8,8 +8,9 @@ for `unload` events need to be synchronous. Both events cannot be cancelled. Example: +**main.ts** + ```ts -// main.ts import "./imported.ts"; const handler = (e: Event): void => { @@ -29,8 +30,11 @@ window.onunload = (e: Event): void => { }; console.log("log from main script"); +``` + +**imported.ts** -// imported.ts +```ts const handler = (e: Event): void => { console.log(`got ${e.type} event in event handler (imported)`); }; @@ -68,3 +72,7 @@ got unload event in event handler (main) All listeners added using `window.addEventListener` were run, but `window.onload` and `window.onunload` defined in `main.ts` overrode handlers defined in `imported.ts`. + +In other words, you can register multiple `window.addEventListener` `"load"` or +`"unload"` events, but only the last loaded `window.onload` or `window.onunload` +events will be executed. diff --git a/docs/runtime/stability.md b/docs/runtime/stability.md index 7fcc0f0b0d5526..298a788d4605c0 100644 --- a/docs/runtime/stability.md +++ b/docs/runtime/stability.md @@ -5,17 +5,28 @@ strive to make code working under 1.0.0 continue to work in future versions. However, not all of Deno's features are ready for production yet. Features which are not ready, because they are still in draft phase, are locked behind the -`--unstable` command line flag. Passing this flag does a few things: +`--unstable` command line flag. + +```shell +deno run --unstable mod_which_uses_unstable_stuff.ts +``` + +Passing this flag does a few things: - It enables the use of unstable APIs during runtime. - It adds the - [`lib.deno.unstable.d.ts`](https://github.com/denoland/deno/blob/master/cli/js/lib.deno.unstable.d.ts) + [`lib.deno.unstable.d.ts`](https://doc.deno.land/https/raw.githubusercontent.com/denoland/deno/master/cli/js/lib.deno.unstable.d.ts) file to the list of TypeScript definitions that are used for type checking. This includes the output of `deno types`. -You should be aware that unstable APIs have probably **not undergone a security +You should be aware that many unstable APIs have **not undergone a security review**, are likely to have **breaking API changes** in the future, and are **not ready for production**. -Furthermore Deno's standard modules (https://deno.land/std/) are not yet stable. -We version the standard modules differently from the CLI to reflect this. +### Standard modules + +Deno's standard modules (https://deno.land/std/) are not yet stable. We +currently version the standard modules differently from the CLI to reflect this. +Note that unlike the `Deno` namespace, the use of the standard modules do not +require the `--unstable` flag (unless the standard module itself makes use of an +unstable Deno feature). diff --git a/docs/runtime/workers.md b/docs/runtime/workers.md index 89541cee75dbe2..110255b8b63bc2 100644 --- a/docs/runtime/workers.md +++ b/docs/runtime/workers.md @@ -25,11 +25,15 @@ requires appropriate permission for this action. For workers using local modules; `--allow-read` permission is required: +**main.ts** + ```ts -// main.ts new Worker("./worker.ts", { type: "module" }); +``` -// worker.ts +**worker.ts** + +```ts console.log("hello world"); self.close(); ``` @@ -44,11 +48,15 @@ hello world For workers using remote modules; `--allow-net` permission is required: +**main.ts** + ```ts -// main.ts new Worker("https://example.com/worker.ts", { type: "module" }); +``` -// worker.ts +**worker.ts** (at https[]()://example.com/worker.ts) + +```ts console.log("hello world"); self.close(); ``` @@ -70,20 +78,27 @@ By default the `Deno` namespace is not available in worker scope. To add the `Deno` namespace pass `deno: true` option when creating new worker: +**main.js** + ```ts -// main.js const worker = new Worker("./worker.js", { type: "module", deno: true }); worker.postMessage({ filename: "./log.txt" }); +``` -// worker.js +**worker.js** + +```ts self.onmessage = async (e) => { const { filename } = e.data; const text = await Deno.readTextFile(filename); console.log(text); self.close(); }; +``` + +**log.txt** -// log.txt +``` hello world ``` diff --git a/docs/testing.md b/docs/testing.md index 5448f6d5823775..67c472b800303e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -35,6 +35,8 @@ You can also test asynchronous code by passing a test function that returns a promise. For this you can use the `async` keyword when defining a function: ```ts +import { delay } from "https://deno.land/std/async/delay.ts"; + Deno.test("async hello world", async () => { const x = 1 + 2; @@ -100,6 +102,6 @@ deno test my_test.ts ``` You can also omit the file name, in which case all tests in the current -directory (recursively) that match the glob `{*_,*.,}test.{js,ts,jsx,tsx}` will -be run. If you pass a directory, all files in the directory that match this glob -will be run. +directory (recursively) that match the glob `{*_,*.,}test.{js,mjs,ts,jsx,tsx}` +will be run. If you pass a directory, all files in the directory that match this +glob will be run. diff --git a/docs/toc.json b/docs/toc.json index 733d1bc00a28f9..dd9793aa812d39 100644 --- a/docs/toc.json +++ b/docs/toc.json @@ -8,6 +8,7 @@ "installation": "Installation", "setup_your_environment": "Setup your environment", "first_steps": "First steps", + "command_line_interface": "Command line interface", "permissions": "Permissions", "typescript": "Using TypeScript", "webassembly": "Using WebAssembly" diff --git a/docs/tools/debugger.md b/docs/tools/debugger.md index e9eccb6f02ac1d..a6aed4711ebe2a 100644 --- a/docs/tools/debugger.md +++ b/docs/tools/debugger.md @@ -14,9 +14,9 @@ first line of code. ### Chrome Devtools -Let's try debugging simple program using Chrome Devtools; for this purpose we'll -use [file_server.ts](https://deno.land/std@v0.50.0/http/file_server.ts) from -`std`; a simple static file server. +Let's try debugging a program using Chrome Devtools; for this purpose we'll use +[file_server.ts](https://deno.land/std@v0.50.0/http/file_server.ts) from `std`; +a static file server. Use `--inspect-brk` flag to break execution on the first line. @@ -83,7 +83,7 @@ Deno can be debugged using VSCode. Official support in plugin is being worked on - https://github.com/denoland/vscode_deno/issues/12 -We can still attach debugger by manually providing simple `launch.json` config: +We can still attach debugger by manually providing a `launch.json` config: ```json { @@ -108,10 +108,10 @@ This time let's try with local source file, create `server.ts`: ```ts import { serve } from "https://deno.land/std@v0.50.0/http/server.ts"; -const s = serve({ port: 8000 }); +const server = serve({ port: 8000 }); console.log("http://localhost:8000/"); -for await (const req of s) { +for await (const req of server) { req.respond({ body: "Hello World\n" }); } ``` @@ -122,6 +122,14 @@ Change `` to `server.ts` and run created configuration: ![VSCode debugger](../images/debugger7.jpg) +### JetBrains IDEs + +You can debug Deno using your JetBrains IDE by right-clicking the file you want +to debug and selecting the `Debug 'Deno: '` option. This will create +a run/debug configuration which has no permission flags set, so to change that +you need to modify the run/debug configuration and change the `Arguments` field +with the required flags. + ### Other Any client that implements Devtools protocol should be able to connect to Deno diff --git a/docs/tools/documentation_generator.md b/docs/tools/documentation_generator.md index 07b0b5c953e382..661fa22e5d200c 100644 --- a/docs/tools/documentation_generator.md +++ b/docs/tools/documentation_generator.md @@ -1,3 +1,33 @@ ## Documentation Generator - +`deno doc` followed by a list of one or more source files will print the JSDoc +documentation for each of the module's **exported** members. Currently, only +exports in the style `export ` and `export ... from ...` are are +supported. + +For example, given a file `add.ts` with the contents: + +```ts +/** + * Adds x and y. + * @param {number} x + * @param {number} y + * @returns {number} Sum of x and y + */ +export function add(x: number, y: number): number { + return x + y; +} +``` + +Running the Deno `doc` command, prints the function's JSDoc comment to `stdout`: + +```shell +deno doc add.ts +function add(x: number, y: number): number + Adds x and y. @param {number} x @param {number} y @returns {number} Sum of x and y +``` + +Use the `--json` flag to output the documentation in JSON format. This JSON +format is consumed by the +[deno doc website](https://github.com/denoland/doc_website) and used to generate +module documentation. diff --git a/std/_util/assert.ts b/std/_util/assert.ts new file mode 100644 index 00000000000000..d797591fe06783 --- /dev/null +++ b/std/_util/assert.ts @@ -0,0 +1,15 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +export class DenoStdInternalError extends Error { + constructor(message: string) { + super(message); + this.name = "DenoStdInternalError"; + } +} + +/** Make an assertion, if not `true`, then throw. */ +export function assert(expr: unknown, msg = ""): asserts expr { + if (!expr) { + throw new DenoStdInternalError(msg); + } +} diff --git a/std/_util/assert_test.ts b/std/_util/assert_test.ts new file mode 100644 index 00000000000000..38aeae91b30287 --- /dev/null +++ b/std/_util/assert_test.ts @@ -0,0 +1,32 @@ +import { assert, DenoStdInternalError } from "./assert.ts"; +import { assertThrows } from "../testing/asserts.ts"; + +const { test } = Deno; + +test({ + name: "assert valid scenario", + fn(): void { + assert(true); + }, +}); + +test({ + name: "assert invalid scenario, no message", + fn(): void { + assertThrows(() => { + assert(false); + }, DenoStdInternalError); + }, +}); +test({ + name: "assert invalid scenario, with message", + fn(): void { + assertThrows( + () => { + assert(false, "Oops! Should be true"); + }, + DenoStdInternalError, + "Oops! Should be true" + ); + }, +}); diff --git a/std/_util/deep_assign.ts b/std/_util/deep_assign.ts index 9034d89bdf4ecd..ca1f0aba11b23d 100644 --- a/std/_util/deep_assign.ts +++ b/std/_util/deep_assign.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; export function deepAssign( target: Record, diff --git a/std/archive/tar.ts b/std/archive/tar.ts index 225bcb7693cd89..52af7bd6942613 100644 --- a/std/archive/tar.ts +++ b/std/archive/tar.ts @@ -28,7 +28,7 @@ */ import { MultiReader } from "../io/readers.ts"; import { BufReader } from "../io/bufio.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; const recordSize = 512; const ustar = "ustar\u000000"; diff --git a/std/datetime/mod.ts b/std/datetime/mod.ts index 258388397d3cb3..7503eccb4338df 100644 --- a/std/datetime/mod.ts +++ b/std/datetime/mod.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; export type DateFormat = "mm-dd-yyyy" | "dd-mm-yyyy" | "yyyy-mm-dd"; diff --git a/std/encoding/README.md b/std/encoding/README.md index c830133ba12ccd..50db5364f94b3b 100644 --- a/std/encoding/README.md +++ b/std/encoding/README.md @@ -29,8 +29,50 @@ writeVarbig(w: Deno.Writer, x: bigint, o: VarbigOptions = {}): Promise ## CSV -- **`parse(input: string | BufReader, opt: ParseCsvOptions): Promise`**: - Read the string/buffer into an +### API + +#### `readMatrix(reader: BufReader, opt: ReadOptions = { comma: ",", trimLeadingSpace: false, lazyQuotes: false }): Promise` + +Parse the CSV from the `reader` with the options provided and return +`string[][]`. + +#### `parse(input: string | BufReader, opt: ParseOptions = { header: false }): Promise`: + +Parse the CSV string/buffer with the options provided. The result of this +function is as follows: + +- If you don't provide both `opt.header` and `opt.parse`, it returns + `string[][]`. +- If you provide `opt.header` but not `opt.parse`, it returns `object[]`. +- If you provide `opt.parse`, it returns an array where each element is the + value returned from `opt.parse`. + +##### `ParseOptions` + +- **`header: boolean | string[] | HeaderOptions[];`**: If a boolean is provided, + the first line will be used as Header definitions. If `string[]` or + `HeaderOptions[]` those names will be used for header definition. +- **`parse?: (input: unknown) => unknown;`**: Parse function for the row, which + will be executed after parsing of all columns. Therefore if you don't provide + header and parse function with headers, input will be `string[]`. + +##### `HeaderOptions` + +- **`name: string;`**: Name of the header to be used as property. +- **`parse?: (input: string) => unknown;`**: Parse function for the column. This + is executed on each entry of the header. This can be combined with the Parse + function of the rows. + +##### `ReadOptions` + +- **`comma?: string;`**: Character which separates values. Default: `','` +- **`comment?: string;`**: Character to start a comment. Default: `'#'` +- **`trimLeadingSpace?: boolean;`**: Flag to trim the leading space of the + value. Default: `false` +- **`lazyQuotes?: boolean;`**: Allow unquoted quote in a quoted field or non + double quoted quotes in quoted field. Default: 'false` +- **`fieldsPerRecord?`**: Enabling the check of fields for each row. If == 0, + first row is used as referral for the number of fields. ### Usage diff --git a/std/encoding/base64.ts b/std/encoding/base64.ts new file mode 100644 index 00000000000000..2f74c8df0d564b --- /dev/null +++ b/std/encoding/base64.ts @@ -0,0 +1,41 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +/** + * Converts given data with base64 encoding + * @param data input to encode + */ +export function encode(data: string | ArrayBuffer): string { + if (typeof data === "string") { + return window.btoa(data); + } else { + const d = new Uint8Array(data); + let dataString = ""; + for (let i = 0; i < d.length; ++i) { + dataString += String.fromCharCode(d[i]); + } + + return window.btoa(dataString); + } +} + +/** + * Converts given base64 encoded data back to original + * @param data input to decode + */ +export function decode(data: string): ArrayBuffer { + const binaryString = decodeString(data); + const binary = new Uint8Array(binaryString.length); + for (let i = 0; i < binary.length; ++i) { + binary[i] = binaryString.charCodeAt(i); + } + + return binary.buffer; +} + +/** + * Decodes data assuming the output is in string type + * @param data input to decode + */ +export function decodeString(data: string): string { + return window.atob(data); +} diff --git a/std/encoding/base64_test.ts b/std/encoding/base64_test.ts new file mode 100644 index 00000000000000..bd559140ada323 --- /dev/null +++ b/std/encoding/base64_test.ts @@ -0,0 +1,47 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +const { test } = Deno; +import { assertEquals } from "../testing/asserts.ts"; +import { encode, decode, decodeString } from "./base64.ts"; + +const testsetString = [ + ["", ""], + ["f", "Zg=="], + ["fo", "Zm8="], + ["foo", "Zm9v"], + ["foob", "Zm9vYg=="], + ["fooba", "Zm9vYmE="], + ["foobar", "Zm9vYmFy"], +]; + +const testsetBinary = [ + [new TextEncoder().encode("\x00"), "AA=="], + [new TextEncoder().encode("\x00\x00"), "AAA="], + [new TextEncoder().encode("\x00\x00\x00"), "AAAA"], + [new TextEncoder().encode("\x00\x00\x00\x00"), "AAAAAA=="], +]; + +test("[encoding/base64] testBase64EncodeString", () => { + for (const [input, output] of testsetString) { + assertEquals(encode(input), output); + } +}); + +test("[encoding/base64] testBase64DecodeString", () => { + for (const [input, output] of testsetString) { + assertEquals(decodeString(output), input); + } +}); + +test("[encoding/base64] testBase64EncodeBinary", () => { + for (const [input, output] of testsetBinary) { + assertEquals(encode(input), output); + } +}); + +test("[encoding/base64] testBase64DecodeBinary", () => { + for (const [input, output] of testsetBinary) { + const outputBinary = new Uint8Array(decode(output as string)); + assertEquals(outputBinary, input as Uint8Array); + } +}); diff --git a/std/encoding/base64url.ts b/std/encoding/base64url.ts new file mode 100644 index 00000000000000..726ea2eb898f66 --- /dev/null +++ b/std/encoding/base64url.ts @@ -0,0 +1,45 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +import { + decode as convertBase64ToArrayBuffer, + encode as convertArrayBufferToBase64, +} from "./base64.ts"; + +/* + * Some variants allow or require omitting the padding '=' signs: + * https://en.wikipedia.org/wiki/Base64#URL_applications + */ +export function addPaddingToBase64url(base64url: string): string { + if (base64url.length % 4 === 2) return base64url + "=="; + if (base64url.length % 4 === 3) return base64url + "="; + if (base64url.length % 4 === 1) + throw new TypeError("Illegal base64url string!"); + return base64url; +} + +function convertBase64urlToBase64(base64url: string): string { + return addPaddingToBase64url(base64url) + .replace(/\-/g, "+") + .replace(/_/g, "/"); +} + +function convertBase64ToBase64url(base64: string): string { + return base64.replace(/=/g, "").replace(/\+/g, "-").replace(/\//g, "_"); +} + +/** + * Converts given data with base64url encoding. + * Removes paddings '='. + * @param data input to encode + */ +export function encode(data: string | ArrayBuffer): string { + return convertBase64ToBase64url(convertArrayBufferToBase64(data)); +} + +/** + * Converts given base64url encoded data back to original + * @param data input to decode + */ +export function decode(data: string): ArrayBuffer { + return convertBase64ToArrayBuffer(convertBase64urlToBase64(data)); +} diff --git a/std/encoding/base64url_test.ts b/std/encoding/base64url_test.ts new file mode 100644 index 00000000000000..2af9096a41bf93 --- /dev/null +++ b/std/encoding/base64url_test.ts @@ -0,0 +1,42 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +const { test } = Deno; +import { assertEquals } from "../testing/asserts.ts"; +import { encode, decode } from "./base64url.ts"; + +const testsetString = [ + ["", ""], + ["f", "Zg"], + ["fo", "Zm8"], + ["foo", "Zm9v"], + ["foob", "Zm9vYg"], + ["fooba", "Zm9vYmE"], + ["foobar", "Zm9vYmFy"], + [">?>d?ß", "Pj8-ZD_f"], +]; + +const testsetBinary = [ + [new TextEncoder().encode("\x00"), "AA"], + [new TextEncoder().encode("\x00\x00"), "AAA"], + [new TextEncoder().encode("\x00\x00\x00"), "AAAA"], + [new TextEncoder().encode("\x00\x00\x00\x00"), "AAAAAA"], +]; + +test("[encoding/base64url] testBase64urlEncodeString", () => { + for (const [input, output] of testsetString) { + assertEquals(encode(input), output); + } +}); + +test("[encoding/base64url] testBase64urlEncodeBinary", () => { + for (const [input, output] of testsetBinary) { + assertEquals(encode(input), output); + } +}); + +test("[encoding/base64ur] testBase64urDecodeBinary", () => { + for (const [input, output] of testsetBinary) { + const outputBinary = new Uint8Array(decode(output as string)); + assertEquals(outputBinary, input as Uint8Array); + } +}); diff --git a/std/encoding/csv.ts b/std/encoding/csv.ts index 3040d492c0331a..5af05453008a4a 100644 --- a/std/encoding/csv.ts +++ b/std/encoding/csv.ts @@ -7,7 +7,7 @@ import { BufReader } from "../io/bufio.ts"; import { TextProtoReader } from "../textproto/mod.ts"; import { StringReader } from "../io/readers.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; const INVALID_RUNE = ["\r", "\n", '"']; @@ -32,7 +32,7 @@ export class ParseError extends Error { * @property trimLeadingSpace - Flag to trim the leading space of the value. * Default: 'false' * @property lazyQuotes - Allow unquoted quote in a quoted field or non double - * quoted quotes in quoted field Default: 'false' + * quoted quotes in quoted field. Default: 'false' * @property fieldsPerRecord - Enabling the check of fields for each row. * If == 0, first row is used as referral for the number of fields. */ @@ -209,6 +209,12 @@ async function readLine(tp: TextProtoReader): Promise { return line; } +/** + * Parse the CSV from the `reader` with the options provided and return `string[][]`. + * + * @param reader provides the CSV data to parse + * @param opt controls the parsing behavior + */ export async function readMatrix( reader: BufReader, opt: ReadOptions = { @@ -253,16 +259,30 @@ export async function readMatrix( } /** + * Parse the CSV string/buffer with the options provided. + * * HeaderOptions provides the column definition * and the parse function for each entry of the * column. */ export interface HeaderOptions { + /** + * Name of the header to be used as property + */ name: string; + /** + * Parse function for the column. + * This is executed on each entry of the header. + * This can be combined with the Parse function of the rows. + */ parse?: (input: string) => unknown; } export interface ParseOptions extends ReadOptions { + /** + * If a boolean is provided, the first line will be used as Header definitions. + * If `string[]` or `HeaderOptions[]` those names will be used for header definition. + */ header: boolean | string[] | HeaderOptions[]; /** Parse function for rows. * Example: @@ -287,6 +307,9 @@ export interface ParseOptions extends ReadOptions { * for columns and rows. * @param input Input to parse. Can be a string or BufReader. * @param opt options of the parser. + * @returns If you don't provide both `opt.header` and `opt.parse`, it returns `string[][]`. + * If you provide `opt.header` but not `opt.parse`, it returns `object[]`. + * If you provide `opt.parse`, it returns an array where each element is the value returned from `opt.parse`. */ export async function parse( input: string | BufReader, diff --git a/std/encoding/csv_test.ts b/std/encoding/csv_test.ts index b3d4ec0c9c9f9a..60d43e35e65e0f 100644 --- a/std/encoding/csv_test.ts +++ b/std/encoding/csv_test.ts @@ -385,7 +385,6 @@ x,,, "#ignore\n".repeat(10000) + "@".repeat(5000) + "," + "*".repeat(5000), Output: [["@".repeat(5000), "*".repeat(5000)]], Comment: "#", - ignore: true, // TODO(#4521) }, { Name: "QuoteWithTrailingCRLF", @@ -455,7 +454,6 @@ x,,, ]; for (const t of testCases) { Deno.test({ - ignore: !!t.ignore, name: `[CSV] ${t.Name}`, async fn(): Promise { let comma = ","; diff --git a/std/encoding/toml.ts b/std/encoding/toml.ts index 7e44bdc18d6132..8a0715ab688929 100644 --- a/std/encoding/toml.ts +++ b/std/encoding/toml.ts @@ -1,6 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import { deepAssign } from "../_util/deep_assign.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; class KeyValuePair { constructor(public key: string, public value: unknown) {} diff --git a/std/examples/catj.ts b/std/examples/catj.ts index bb2e9051b633b5..93fc68a1d68cdf 100644 --- a/std/examples/catj.ts +++ b/std/examples/catj.ts @@ -1,4 +1,4 @@ -#!/usr/bin/env -S deno --allow-read +#!/usr/bin/env -S deno run --allow-read // Ported from: https://github.com/soheilpro/catj // Copyright (c) 2014 Soheil Rashidi // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. diff --git a/std/examples/gist.ts b/std/examples/gist.ts index 5d66dbdafed7a7..555ec8056485ad 100755 --- a/std/examples/gist.ts +++ b/std/examples/gist.ts @@ -1,4 +1,4 @@ -#!/usr/bin/env -S deno --allow-net --allow-env +#!/usr/bin/env -S deno run --allow-net --allow-env // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. // A program to post files to gist.github.com. Use the following to install it: // deno install -f --allow-env --allow-read --allow-net=api.github.com https://deno.land/std/examples/gist.ts diff --git a/std/examples/tests/cat_test.ts b/std/examples/tests/cat_test.ts index 4633c5750c93ba..8fb1244603bae2 100644 --- a/std/examples/tests/cat_test.ts +++ b/std/examples/tests/cat_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assertStrictEq } from "../../testing/asserts.ts"; +import { assertStrictEquals } from "../../testing/asserts.ts"; Deno.test("[examples/cat] print multiple files", async () => { const decoder = new TextDecoder(); @@ -19,7 +19,7 @@ Deno.test("[examples/cat] print multiple files", async () => { try { const output = await process.output(); const actual = decoder.decode(output).trim(); - assertStrictEq(actual, "Hello\nWorld"); + assertStrictEquals(actual, "Hello\nWorld"); } finally { process.close(); } diff --git a/std/examples/tests/catj_test.ts b/std/examples/tests/catj_test.ts index c3eb61f515835d..a859db0c15b7b4 100644 --- a/std/examples/tests/catj_test.ts +++ b/std/examples/tests/catj_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assertStrictEq } from "../../testing/asserts.ts"; +import { assertStrictEquals } from "../../testing/asserts.ts"; Deno.test("[examples/catj] print an array", async () => { const decoder = new TextDecoder(); @@ -15,7 +15,7 @@ Deno.test("[examples/catj] print an array", async () => { '.[2].array[1] = "bar"', ].join("\n"); - assertStrictEq(actual, expected); + assertStrictEquals(actual, expected); } finally { process.stdin!.close(); process.close(); @@ -34,7 +34,7 @@ Deno.test("[examples/catj] print an object", async () => { '.array[0].message = "hello"', ].join("\n"); - assertStrictEq(actual, expected); + assertStrictEquals(actual, expected); } finally { process.stdin!.close(); process.close(); @@ -52,7 +52,7 @@ Deno.test("[examples/catj] print multiple files", async () => { const actual = decoder.decode(output).trim(); const expected = ['.message = "hello"', ".[0] = 1", ".[1] = 2"].join("\n"); - assertStrictEq(actual, expected); + assertStrictEquals(actual, expected); } finally { process.stdin!.close(); process.close(); @@ -69,7 +69,7 @@ Deno.test("[examples/catj] read from stdin", async () => { const output = await process.output(); const actual = decoder.decode(output).trim(); - assertStrictEq(actual, '.foo = "bar"'); + assertStrictEquals(actual, '.foo = "bar"'); } finally { process.close(); } diff --git a/std/examples/tests/colors_test.ts b/std/examples/tests/colors_test.ts index 90c469c009bd43..1a3e4f41834d77 100644 --- a/std/examples/tests/colors_test.ts +++ b/std/examples/tests/colors_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assertStrictEq } from "../../testing/asserts.ts"; +import { assertStrictEquals } from "../../testing/asserts.ts"; Deno.test("[examples/colors] print a colored text", async () => { const decoder = new TextDecoder(); @@ -12,7 +12,7 @@ Deno.test("[examples/colors] print a colored text", async () => { const output = await process.output(); const actual = decoder.decode(output).trim(); const expected = "Hello world!"; - assertStrictEq(actual, expected); + assertStrictEquals(actual, expected); } finally { process.close(); } diff --git a/std/examples/tests/curl_test.ts b/std/examples/tests/curl_test.ts index 69e0e52d6b8907..4449d11ea5f031 100644 --- a/std/examples/tests/curl_test.ts +++ b/std/examples/tests/curl_test.ts @@ -1,6 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import { serve } from "../../http/server.ts"; -import { assertStrictEq } from "../../testing/asserts.ts"; +import { assertStrictEquals } from "../../testing/asserts.ts"; Deno.test({ name: "[examples/curl] send a request to a specified url", @@ -30,7 +30,7 @@ Deno.test({ const actual = decoder.decode(output).trim(); const expected = "Hello world"; - assertStrictEq(actual, expected); + assertStrictEquals(actual, expected); } finally { server.close(); process.close(); diff --git a/std/examples/tests/echo_server_test.ts b/std/examples/tests/echo_server_test.ts index 3e60961909eee4..475b0f73f1f90d 100644 --- a/std/examples/tests/echo_server_test.ts +++ b/std/examples/tests/echo_server_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assertStrictEq, assertNotEquals } from "../../testing/asserts.ts"; +import { assertStrictEquals, assertNotEquals } from "../../testing/asserts.ts"; import { BufReader, ReadLineResult } from "../../io/bufio.ts"; Deno.test("[examples/echo_server]", async () => { @@ -17,7 +17,7 @@ Deno.test("[examples/echo_server]", async () => { const message = await processReader.readLine(); assertNotEquals(message, null); - assertStrictEq( + assertStrictEquals( decoder.decode((message as ReadLineResult).line).trim(), "Listening on 0.0.0.0:8080" ); @@ -35,7 +35,7 @@ Deno.test("[examples/echo_server]", async () => { .trim(); const expectedResponse = "Hello echo_server"; - assertStrictEq(actualResponse, expectedResponse); + assertStrictEquals(actualResponse, expectedResponse); } finally { conn?.close(); process.stdout!.close(); diff --git a/std/examples/tests/welcome_test.ts b/std/examples/tests/welcome_test.ts index d508773c555433..d585211a1898ff 100644 --- a/std/examples/tests/welcome_test.ts +++ b/std/examples/tests/welcome_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assertStrictEq } from "../../testing/asserts.ts"; +import { assertStrictEquals } from "../../testing/asserts.ts"; Deno.test("[examples/welcome] print a welcome message", async () => { const decoder = new TextDecoder(); @@ -12,7 +12,7 @@ Deno.test("[examples/welcome] print a welcome message", async () => { const output = await process.output(); const actual = decoder.decode(output).trim(); const expected = "Welcome to Deno 🦕"; - assertStrictEq(actual, expected); + assertStrictEquals(actual, expected); } finally { process.close(); } diff --git a/std/examples/tests/xeval_test.ts b/std/examples/tests/xeval_test.ts index eeeac7731acab7..f86e2786660e62 100644 --- a/std/examples/tests/xeval_test.ts +++ b/std/examples/tests/xeval_test.ts @@ -1,25 +1,29 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import { xeval } from "../xeval.ts"; -import { stringsReader } from "../../io/util.ts"; +import { StringReader } from "../../io/readers.ts"; import { decode, encode } from "../../encoding/utf8.ts"; import { assertEquals, - assertStrContains, + assertStringContains, assert, } from "../../testing/asserts.ts"; const { execPath, run } = Deno; Deno.test("xevalSuccess", async function (): Promise { const chunks: string[] = []; - await xeval(stringsReader("a\nb\nc"), ($): number => chunks.push($)); + await xeval(new StringReader("a\nb\nc"), ($): number => chunks.push($)); assertEquals(chunks, ["a", "b", "c"]); }); Deno.test("xevalDelimiter", async function (): Promise { const chunks: string[] = []; - await xeval(stringsReader("!MADMADAMADAM!"), ($): number => chunks.push($), { - delimiter: "MADAM", - }); + await xeval( + new StringReader("!MADMADAMADAM!"), + ($): number => chunks.push($), + { + delimiter: "MADAM", + } + ); assertEquals(chunks, ["!MAD", "ADAM!"]); }); @@ -52,6 +56,6 @@ Deno.test("xevalCliSyntaxError", async function (): Promise { }); assertEquals(await p.status(), { code: 1, success: false }); assertEquals(decode(await p.output()), ""); - assertStrContains(decode(await p.stderrOutput()), "Uncaught SyntaxError"); + assertStringContains(decode(await p.stderrOutput()), "Uncaught SyntaxError"); p.close(); }); diff --git a/std/flags/mod.ts b/std/flags/mod.ts index e3680087de601f..5c8fcc0d91e275 100644 --- a/std/flags/mod.ts +++ b/std/flags/mod.ts @@ -1,6 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; export interface Args { /** Contains all the arguments that didn't have an option associated with diff --git a/std/fmt/colors.ts b/std/fmt/colors.ts index 6a06af20e057e6..a020657d990d91 100644 --- a/std/fmt/colors.ts +++ b/std/fmt/colors.ts @@ -1,6 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -/** - * A module to print ANSI terminal colors. Inspired by chalk, kleur, and colors +/** A module to print ANSI terminal colors. Inspired by chalk, kleur, and colors * on npm. * * ``` @@ -10,8 +9,10 @@ * * This module supports `NO_COLOR` environmental variable disabling any coloring * if `NO_COLOR` is set. - */ -const { noColor } = Deno; + * + * This module is browser compatible. */ + +const noColor = globalThis.Deno?.noColor ?? true; interface Code { open: string; diff --git a/std/fs/copy.ts b/std/fs/copy.ts index d45ac17c95f312..269340e8504603 100644 --- a/std/fs/copy.ts +++ b/std/fs/copy.ts @@ -2,7 +2,7 @@ import * as path from "../path/mod.ts"; import { ensureDir, ensureDirSync } from "./ensure_dir.ts"; import { isSubdir, getFileInfoType } from "./_util.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; const isWindows = Deno.build.os === "windows"; diff --git a/std/fs/empty_dir_test.ts b/std/fs/empty_dir_test.ts index aff16f6a5ce0c6..e0e4c58fa94172 100644 --- a/std/fs/empty_dir_test.ts +++ b/std/fs/empty_dir_test.ts @@ -2,7 +2,7 @@ import { assert, assertEquals, - assertStrContains, + assertStringContains, assertThrows, assertThrowsAsync, } from "../testing/asserts.ts"; @@ -228,7 +228,7 @@ for (const s of scenes) { assert(p.stdout); const output = await p.output(); p.close(); - assertStrContains(new TextDecoder().decode(output), s.output); + assertStringContains(new TextDecoder().decode(output), s.output); } catch (err) { await Deno.remove(testfolder, { recursive: true }); throw err; diff --git a/std/fs/exists_test.ts b/std/fs/exists_test.ts index 191289d88d7a30..8b584d86181101 100644 --- a/std/fs/exists_test.ts +++ b/std/fs/exists_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { assertEquals, assertStrContains } from "../testing/asserts.ts"; +import { assertEquals, assertStringContains } from "../testing/asserts.ts"; import * as path from "../path/mod.ts"; import { exists, existsSync } from "./exists.ts"; @@ -130,7 +130,7 @@ for (const s of scenes) { const output = await p.output(); p.close(); - assertStrContains(new TextDecoder().decode(output), s.output); + assertStringContains(new TextDecoder().decode(output), s.output); }); // done } diff --git a/std/fs/expand_glob.ts b/std/fs/expand_glob.ts index e5abdabcf3dbcf..4b7b118854e9ed 100644 --- a/std/fs/expand_glob.ts +++ b/std/fs/expand_glob.ts @@ -14,7 +14,7 @@ import { walk, walkSync, } from "./walk.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; const { cwd } = Deno; type FileInfo = Deno.FileInfo; diff --git a/std/fs/expand_glob_test.ts b/std/fs/expand_glob_test.ts index 24885530bbd2cc..7d60d024ed25d7 100644 --- a/std/fs/expand_glob_test.ts +++ b/std/fs/expand_glob_test.ts @@ -1,6 +1,10 @@ const { cwd, execPath, run } = Deno; import { decode } from "../encoding/utf8.ts"; -import { assert, assertEquals, assertStrContains } from "../testing/asserts.ts"; +import { + assert, + assertEquals, + assertStringContains, +} from "../testing/asserts.ts"; import { join, joinGlobs, @@ -122,7 +126,7 @@ Deno.test("expandGlobPermError", async function (): Promise { }); assertEquals(await p.status(), { code: 1, success: false }); assertEquals(decode(await p.output()), ""); - assertStrContains( + assertStringContains( decode(await p.stderrOutput()), "Uncaught PermissionDenied" ); diff --git a/std/fs/walk.ts b/std/fs/walk.ts index ccd5097a496d8e..553e52b2ece742 100644 --- a/std/fs/walk.ts +++ b/std/fs/walk.ts @@ -1,7 +1,7 @@ // Documentation and interface for walk were adapted from Go // https://golang.org/pkg/path/filepath/#Walk // Copyright 2009 The Go Authors. All rights reserved. BSD license. -import { unimplemented, assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; import { basename, join, normalize } from "../path/mod.ts"; const { readDir, readDirSync, stat, statSync } = Deno; @@ -107,7 +107,7 @@ export async function* walk( if (entry.isSymlink) { if (followSymlinks) { // TODO(ry) Re-enable followSymlinks. - unimplemented(); + throw new Error("unimplemented"); } else { continue; } @@ -159,7 +159,7 @@ export function* walkSync( for (const entry of readDirSync(root)) { if (entry.isSymlink) { if (followSymlinks) { - unimplemented(); + throw new Error("unimplemented"); } else { continue; } diff --git a/std/hash/README.md b/std/hash/README.md new file mode 100644 index 00000000000000..5e511a97cfe531 --- /dev/null +++ b/std/hash/README.md @@ -0,0 +1,66 @@ +# std/hash + +## MD5 + +**Uses:** + +```ts +import { Md5 } from "https://deno.land/std/hash/md5.ts"; + +const md5 = new Md5(); +const md5Instance = md5.update("中文"); // return instance of `Md5` +console.log(md5Instance instanceof Md5); // true +console.log(md5Instance.toString()); // a7bac2239fcdcb3a067903d8077c4a07 +``` + +Calling `update` method, It will update internal state based on the input +provided. Once you call `md5Instance.toString()`, it will return the +`hash string`. You can provide format as `hash` or `base64`. The default format +is `hex`. + +**sample:** + +```ts +console.log(md5Instance.toString("base64")); // MNgWOD+FHGO3Fff/HDCY2w== +``` + +## SHA1 + +**Uses:** + +Creating `sha1` hash is simple. You can use `Sha1` class instance and update the +digest. Calling `hex` method will return the sha1 in hex value. You can also use +`toString` method. + +```ts +import { Sha1 } from "https://deno.land/std/hash/sha1.ts"; + +const sha1 = new Sha1().update("中文"); +console.log(sha1.hex()); // 7be2d2d20c106eee0836c9bc2b939890a78e8fb3 +console.log(sha1.toString()); // same as above +``` + +## Sha256 and HmacSha256 + +**Uses:** + +Creating `Sha256` hash is simple. You can use `Sha256` class instance and update +the digest. Calling the `hex` method will return the sha256 in `hex` value. You +can also use the `toString` method. + +**Note:** For `HmacSha256`, you can pass the secret `key` while creating an +instance of the object. + +```ts +import { Sha256, HmacSha256 } from "https://deno.land/std/hash/sha256.ts"; + +const sha256 = new Sha256().update("中文"); +console.log(sha256.hex()); +console.log(sha256.toString()); // Same as above + +const key = "Hi There"; +const hmac = new HmacSha256(key).update("中文"); + +console.log(hmac.hex()); +console.log(hmac.toString()); // Same as above +``` diff --git a/std/hash/_sha3/keccak.ts b/std/hash/_sha3/keccak.ts new file mode 100644 index 00000000000000..67cccca3c0e7b2 --- /dev/null +++ b/std/hash/_sha3/keccak.ts @@ -0,0 +1,52 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +import { Sponge } from "./sponge.ts"; +import { keccakf } from "./keccakf.ts"; + +/** Keccak-224 hash */ +export class Keccak224 extends Sponge { + constructor() { + super({ + bitsize: 224, + rate: 144, + dsbyte: 1, + permutator: keccakf, + }); + } +} + +/** Keccak-256 hash */ +export class Keccak256 extends Sponge { + constructor() { + super({ + bitsize: 256, + rate: 136, + dsbyte: 1, + permutator: keccakf, + }); + } +} + +/** Keccak-384 hash */ +export class Keccak384 extends Sponge { + constructor() { + super({ + bitsize: 384, + rate: 104, + dsbyte: 1, + permutator: keccakf, + }); + } +} + +/** Keccak-512 hash */ +export class Keccak512 extends Sponge { + constructor() { + super({ + bitsize: 512, + rate: 72, + dsbyte: 1, + permutator: keccakf, + }); + } +} diff --git a/std/hash/_sha3/keccakf.ts b/std/hash/_sha3/keccakf.ts new file mode 100644 index 00000000000000..8db90b1a4b4eae --- /dev/null +++ b/std/hash/_sha3/keccakf.ts @@ -0,0 +1,790 @@ +// Ported from Go: +// https://github.com/golang/crypto/blob/master/sha3/keccakf.go +// Copyright 2011 The Go Authors. All rights reserved. BSD license. +// https://github.com/golang/go/blob/master/LICENSE +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +const KECCAK_ROUNDS = 24; +const KECCAK_RC: number[] = [ + 0x1, + 0x0, + 0x8082, + 0x0, + 0x808a, + 0x80000000, + 0x80008000, + 0x80000000, + 0x808b, + 0x0, + 0x80000001, + 0x0, + 0x80008081, + 0x80000000, + 0x8009, + 0x80000000, + 0x8a, + 0x0, + 0x88, + 0x0, + 0x80008009, + 0x0, + 0x8000000a, + 0x0, + 0x8000808b, + 0x0, + 0x8b, + 0x80000000, + 0x8089, + 0x80000000, + 0x8003, + 0x80000000, + 0x8002, + 0x80000000, + 0x80, + 0x80000000, + 0x800a, + 0x0, + 0x8000000a, + 0x80000000, + 0x80008081, + 0x80000000, + 0x8080, + 0x80000000, + 0x80000001, + 0x0, + 0x80008008, + 0x80000000, +]; + +/** keccak1600 permutation function */ +export function keccakf(state: Uint8Array): void { + const s = new Uint32Array(state.buffer); + let bc0 = 0; + let bc1 = 0; + let bc2 = 0; + let bc3 = 0; + let bc4 = 0; + let bc5 = 0; + let bc6 = 0; + let bc7 = 0; + let bc8 = 0; + let bc9 = 0; + let d0 = 0; + let d1 = 0; + let d2 = 0; + let d3 = 0; + let d4 = 0; + let d5 = 0; + let d6 = 0; + let d7 = 0; + let d8 = 0; + let d9 = 0; + let t0 = 0; + let t1 = 0; + + for (let n = 0; n < KECCAK_ROUNDS * 2; n += 8) { + // Round 1 + bc0 = s[0] ^ s[10] ^ s[20] ^ s[30] ^ s[40]; + bc1 = s[1] ^ s[11] ^ s[21] ^ s[31] ^ s[41]; + bc2 = s[2] ^ s[12] ^ s[22] ^ s[32] ^ s[42]; + bc3 = s[3] ^ s[13] ^ s[23] ^ s[33] ^ s[43]; + bc4 = s[4] ^ s[14] ^ s[24] ^ s[34] ^ s[44]; + bc5 = s[5] ^ s[15] ^ s[25] ^ s[35] ^ s[45]; + bc6 = s[6] ^ s[16] ^ s[26] ^ s[36] ^ s[46]; + bc7 = s[7] ^ s[17] ^ s[27] ^ s[37] ^ s[47]; + bc8 = s[8] ^ s[18] ^ s[28] ^ s[38] ^ s[48]; + bc9 = s[9] ^ s[19] ^ s[29] ^ s[39] ^ s[49]; + + d0 = bc8 ^ ((bc2 << 1) | (bc3 >>> 31)); + d1 = bc9 ^ ((bc3 << 1) | (bc2 >>> 31)); + d2 = bc0 ^ ((bc4 << 1) | (bc5 >>> 31)); + d3 = bc1 ^ ((bc5 << 1) | (bc4 >>> 31)); + d4 = bc2 ^ ((bc6 << 1) | (bc7 >>> 31)); + d5 = bc3 ^ ((bc7 << 1) | (bc6 >>> 31)); + d6 = bc4 ^ ((bc8 << 1) | (bc9 >>> 31)); + d7 = bc5 ^ ((bc9 << 1) | (bc8 >>> 31)); + d8 = bc6 ^ ((bc0 << 1) | (bc1 >>> 31)); + d9 = bc7 ^ ((bc1 << 1) | (bc0 >>> 31)); + + bc0 = s[0] ^ d0; + bc1 = s[1] ^ d1; + t0 = s[12] ^ d2; + t1 = s[13] ^ d3; + bc2 = (t1 << 12) | (t0 >>> 20); + bc3 = (t0 << 12) | (t1 >>> 20); + t0 = s[24] ^ d4; + t1 = s[25] ^ d5; + bc4 = (t1 << 11) | (t0 >>> 21); + bc5 = (t0 << 11) | (t1 >>> 21); + t0 = s[36] ^ d6; + t1 = s[37] ^ d7; + bc6 = (t0 << 21) | (t1 >>> 11); + bc7 = (t1 << 21) | (t0 >>> 11); + t0 = s[48] ^ d8; + t1 = s[49] ^ d9; + bc8 = (t0 << 14) | (t1 >>> 18); + bc9 = (t1 << 14) | (t0 >>> 18); + s[0] = bc0 ^ (bc4 & ~bc2) ^ KECCAK_RC[n]; + s[1] = bc1 ^ (bc5 & ~bc3) ^ KECCAK_RC[n + 1]; + s[12] = bc2 ^ (bc6 & ~bc4); + s[13] = bc3 ^ (bc7 & ~bc5); + s[24] = bc4 ^ (bc8 & ~bc6); + s[25] = bc5 ^ (bc9 & ~bc7); + s[36] = bc6 ^ (bc0 & ~bc8); + s[37] = bc7 ^ (bc1 & ~bc9); + s[48] = bc8 ^ (bc2 & ~bc0); + s[49] = bc9 ^ (bc3 & ~bc1); + + t0 = s[20] ^ d0; + t1 = s[21] ^ d1; + bc4 = (t0 << 3) | (t1 >>> 29); + bc5 = (t1 << 3) | (t0 >>> 29); + t0 = s[32] ^ d2; + t1 = s[33] ^ d3; + bc6 = (t1 << 13) | (t0 >>> 19); + bc7 = (t0 << 13) | (t1 >>> 19); + t0 = s[44] ^ d4; + t1 = s[45] ^ d5; + bc8 = (t1 << 29) | (t0 >>> 3); + bc9 = (t0 << 29) | (t1 >>> 3); + t0 = s[6] ^ d6; + t1 = s[7] ^ d7; + bc0 = (t0 << 28) | (t1 >>> 4); + bc1 = (t1 << 28) | (t0 >>> 4); + t0 = s[18] ^ d8; + t1 = s[19] ^ d9; + bc2 = (t0 << 20) | (t1 >>> 12); + bc3 = (t1 << 20) | (t0 >>> 12); + s[20] = bc0 ^ (bc4 & ~bc2); + s[21] = bc1 ^ (bc5 & ~bc3); + s[32] = bc2 ^ (bc6 & ~bc4); + s[33] = bc3 ^ (bc7 & ~bc5); + s[44] = bc4 ^ (bc8 & ~bc6); + s[45] = bc5 ^ (bc9 & ~bc7); + s[6] = bc6 ^ (bc0 & ~bc8); + s[7] = bc7 ^ (bc1 & ~bc9); + s[18] = bc8 ^ (bc2 & ~bc0); + s[19] = bc9 ^ (bc3 & ~bc1); + + t0 = s[40] ^ d0; + t1 = s[41] ^ d1; + bc8 = (t0 << 18) | (t1 >>> 14); + bc9 = (t1 << 18) | (t0 >>> 14); + t0 = s[2] ^ d2; + t1 = s[3] ^ d3; + bc0 = (t0 << 1) | (t1 >>> 31); + bc1 = (t1 << 1) | (t0 >>> 31); + t0 = s[14] ^ d4; + t1 = s[15] ^ d5; + bc2 = (t0 << 6) | (t1 >>> 26); + bc3 = (t1 << 6) | (t0 >>> 26); + t0 = s[26] ^ d6; + t1 = s[27] ^ d7; + bc4 = (t0 << 25) | (t1 >>> 7); + bc5 = (t1 << 25) | (t0 >>> 7); + t0 = s[38] ^ d8; + t1 = s[39] ^ d9; + bc6 = (t0 << 8) | (t1 >>> 24); + bc7 = (t1 << 8) | (t0 >>> 24); + s[40] = bc0 ^ (bc4 & ~bc2); + s[41] = bc1 ^ (bc5 & ~bc3); + s[2] = bc2 ^ (bc6 & ~bc4); + s[3] = bc3 ^ (bc7 & ~bc5); + s[14] = bc4 ^ (bc8 & ~bc6); + s[15] = bc5 ^ (bc9 & ~bc7); + s[26] = bc6 ^ (bc0 & ~bc8); + s[27] = bc7 ^ (bc1 & ~bc9); + s[38] = bc8 ^ (bc2 & ~bc0); + s[39] = bc9 ^ (bc3 & ~bc1); + + t0 = s[10] ^ d0; + t1 = s[11] ^ d1; + bc2 = (t1 << 4) | (t0 >>> 28); + bc3 = (t0 << 4) | (t1 >>> 28); + t0 = s[22] ^ d2; + t1 = s[23] ^ d3; + bc4 = (t0 << 10) | (t1 >>> 22); + bc5 = (t1 << 10) | (t0 >>> 22); + t0 = s[34] ^ d4; + t1 = s[35] ^ d5; + bc6 = (t0 << 15) | (t1 >>> 17); + bc7 = (t1 << 15) | (t0 >>> 17); + t0 = s[46] ^ d6; + t1 = s[47] ^ d7; + bc8 = (t1 << 24) | (t0 >>> 8); + bc9 = (t0 << 24) | (t1 >>> 8); + t0 = s[8] ^ d8; + t1 = s[9] ^ d9; + bc0 = (t0 << 27) | (t1 >>> 5); + bc1 = (t1 << 27) | (t0 >>> 5); + s[10] = bc0 ^ (bc4 & ~bc2); + s[11] = bc1 ^ (bc5 & ~bc3); + s[22] = bc2 ^ (bc6 & ~bc4); + s[23] = bc3 ^ (bc7 & ~bc5); + s[34] = bc4 ^ (bc8 & ~bc6); + s[35] = bc5 ^ (bc9 & ~bc7); + s[46] = bc6 ^ (bc0 & ~bc8); + s[47] = bc7 ^ (bc1 & ~bc9); + s[8] = bc8 ^ (bc2 & ~bc0); + s[9] = bc9 ^ (bc3 & ~bc1); + + t0 = s[30] ^ d0; + t1 = s[31] ^ d1; + bc6 = (t1 << 9) | (t0 >>> 23); + bc7 = (t0 << 9) | (t1 >>> 23); + t0 = s[42] ^ d2; + t1 = s[43] ^ d3; + bc8 = (t0 << 2) | (t1 >>> 30); + bc9 = (t1 << 2) | (t0 >>> 30); + t0 = s[4] ^ d4; + t1 = s[5] ^ d5; + bc0 = (t1 << 30) | (t0 >>> 2); + bc1 = (t0 << 30) | (t1 >>> 2); + t0 = s[16] ^ d6; + t1 = s[17] ^ d7; + bc2 = (t1 << 23) | (t0 >>> 9); + bc3 = (t0 << 23) | (t1 >>> 9); + t0 = s[28] ^ d8; + t1 = s[29] ^ d9; + bc4 = (t1 << 7) | (t0 >>> 25); + bc5 = (t0 << 7) | (t1 >>> 25); + s[30] = bc0 ^ (bc4 & ~bc2); + s[31] = bc1 ^ (bc5 & ~bc3); + s[42] = bc2 ^ (bc6 & ~bc4); + s[43] = bc3 ^ (bc7 & ~bc5); + s[4] = bc4 ^ (bc8 & ~bc6); + s[5] = bc5 ^ (bc9 & ~bc7); + s[16] = bc6 ^ (bc0 & ~bc8); + s[17] = bc7 ^ (bc1 & ~bc9); + s[28] = bc8 ^ (bc2 & ~bc0); + s[29] = bc9 ^ (bc3 & ~bc1); + + // Round 2 + bc0 = s[0] ^ s[10] ^ s[20] ^ s[30] ^ s[40]; + bc1 = s[1] ^ s[11] ^ s[21] ^ s[31] ^ s[41]; + bc2 = s[2] ^ s[12] ^ s[22] ^ s[32] ^ s[42]; + bc3 = s[3] ^ s[13] ^ s[23] ^ s[33] ^ s[43]; + bc4 = s[4] ^ s[14] ^ s[24] ^ s[34] ^ s[44]; + bc5 = s[5] ^ s[15] ^ s[25] ^ s[35] ^ s[45]; + bc6 = s[6] ^ s[16] ^ s[26] ^ s[36] ^ s[46]; + bc7 = s[7] ^ s[17] ^ s[27] ^ s[37] ^ s[47]; + bc8 = s[8] ^ s[18] ^ s[28] ^ s[38] ^ s[48]; + bc9 = s[9] ^ s[19] ^ s[29] ^ s[39] ^ s[49]; + + d0 = bc8 ^ ((bc2 << 1) | (bc3 >>> 31)); + d1 = bc9 ^ ((bc3 << 1) | (bc2 >>> 31)); + d2 = bc0 ^ ((bc4 << 1) | (bc5 >>> 31)); + d3 = bc1 ^ ((bc5 << 1) | (bc4 >>> 31)); + d4 = bc2 ^ ((bc6 << 1) | (bc7 >>> 31)); + d5 = bc3 ^ ((bc7 << 1) | (bc6 >>> 31)); + d6 = bc4 ^ ((bc8 << 1) | (bc9 >>> 31)); + d7 = bc5 ^ ((bc9 << 1) | (bc8 >>> 31)); + d8 = bc6 ^ ((bc0 << 1) | (bc1 >>> 31)); + d9 = bc7 ^ ((bc1 << 1) | (bc0 >>> 31)); + + bc0 = s[0] ^ d0; + bc1 = s[1] ^ d1; + t0 = s[32] ^ d2; + t1 = s[33] ^ d3; + bc2 = (t1 << 12) | (t0 >>> 20); + bc3 = (t0 << 12) | (t1 >>> 20); + t0 = s[14] ^ d4; + t1 = s[15] ^ d5; + bc4 = (t1 << 11) | (t0 >>> 21); + bc5 = (t0 << 11) | (t1 >>> 21); + t0 = s[46] ^ d6; + t1 = s[47] ^ d7; + bc6 = (t0 << 21) | (t1 >>> 11); + bc7 = (t1 << 21) | (t0 >>> 11); + t0 = s[28] ^ d8; + t1 = s[29] ^ d9; + bc8 = (t0 << 14) | (t1 >>> 18); + bc9 = (t1 << 14) | (t0 >>> 18); + s[0] = bc0 ^ (bc4 & ~bc2) ^ KECCAK_RC[n + 2]; + s[1] = bc1 ^ (bc5 & ~bc3) ^ KECCAK_RC[n + 3]; + s[32] = bc2 ^ (bc6 & ~bc4); + s[33] = bc3 ^ (bc7 & ~bc5); + s[14] = bc4 ^ (bc8 & ~bc6); + s[15] = bc5 ^ (bc9 & ~bc7); + s[46] = bc6 ^ (bc0 & ~bc8); + s[47] = bc7 ^ (bc1 & ~bc9); + s[28] = bc8 ^ (bc2 & ~bc0); + s[29] = bc9 ^ (bc3 & ~bc1); + + t0 = s[40] ^ d0; + t1 = s[41] ^ d1; + bc4 = (t0 << 3) | (t1 >>> 29); + bc5 = (t1 << 3) | (t0 >>> 29); + t0 = s[22] ^ d2; + t1 = s[23] ^ d3; + bc6 = (t1 << 13) | (t0 >>> 19); + bc7 = (t0 << 13) | (t1 >>> 19); + t0 = s[4] ^ d4; + t1 = s[5] ^ d5; + bc8 = (t1 << 29) | (t0 >>> 3); + bc9 = (t0 << 29) | (t1 >>> 3); + t0 = s[36] ^ d6; + t1 = s[37] ^ d7; + bc0 = (t0 << 28) | (t1 >>> 4); + bc1 = (t1 << 28) | (t0 >>> 4); + t0 = s[18] ^ d8; + t1 = s[19] ^ d9; + bc2 = (t0 << 20) | (t1 >>> 12); + bc3 = (t1 << 20) | (t0 >>> 12); + s[40] = bc0 ^ (bc4 & ~bc2); + s[41] = bc1 ^ (bc5 & ~bc3); + s[22] = bc2 ^ (bc6 & ~bc4); + s[23] = bc3 ^ (bc7 & ~bc5); + s[4] = bc4 ^ (bc8 & ~bc6); + s[5] = bc5 ^ (bc9 & ~bc7); + s[36] = bc6 ^ (bc0 & ~bc8); + s[37] = bc7 ^ (bc1 & ~bc9); + s[18] = bc8 ^ (bc2 & ~bc0); + s[19] = bc9 ^ (bc3 & ~bc1); + + t0 = s[30] ^ d0; + t1 = s[31] ^ d1; + bc8 = (t0 << 18) | (t1 >>> 14); + bc9 = (t1 << 18) | (t0 >>> 14); + t0 = s[12] ^ d2; + t1 = s[13] ^ d3; + bc0 = (t0 << 1) | (t1 >>> 31); + bc1 = (t1 << 1) | (t0 >>> 31); + t0 = s[44] ^ d4; + t1 = s[45] ^ d5; + bc2 = (t0 << 6) | (t1 >>> 26); + bc3 = (t1 << 6) | (t0 >>> 26); + t0 = s[26] ^ d6; + t1 = s[27] ^ d7; + bc4 = (t0 << 25) | (t1 >>> 7); + bc5 = (t1 << 25) | (t0 >>> 7); + t0 = s[8] ^ d8; + t1 = s[9] ^ d9; + bc6 = (t0 << 8) | (t1 >>> 24); + bc7 = (t1 << 8) | (t0 >>> 24); + s[30] = bc0 ^ (bc4 & ~bc2); + s[31] = bc1 ^ (bc5 & ~bc3); + s[12] = bc2 ^ (bc6 & ~bc4); + s[13] = bc3 ^ (bc7 & ~bc5); + s[44] = bc4 ^ (bc8 & ~bc6); + s[45] = bc5 ^ (bc9 & ~bc7); + s[26] = bc6 ^ (bc0 & ~bc8); + s[27] = bc7 ^ (bc1 & ~bc9); + s[8] = bc8 ^ (bc2 & ~bc0); + s[9] = bc9 ^ (bc3 & ~bc1); + + t0 = s[20] ^ d0; + t1 = s[21] ^ d1; + bc2 = (t1 << 4) | (t0 >>> 28); + bc3 = (t0 << 4) | (t1 >>> 28); + t0 = s[2] ^ d2; + t1 = s[3] ^ d3; + bc4 = (t0 << 10) | (t1 >>> 22); + bc5 = (t1 << 10) | (t0 >>> 22); + t0 = s[34] ^ d4; + t1 = s[35] ^ d5; + bc6 = (t0 << 15) | (t1 >>> 17); + bc7 = (t1 << 15) | (t0 >>> 17); + t0 = s[16] ^ d6; + t1 = s[17] ^ d7; + bc8 = (t1 << 24) | (t0 >>> 8); + bc9 = (t0 << 24) | (t1 >>> 8); + t0 = s[48] ^ d8; + t1 = s[49] ^ d9; + bc0 = (t0 << 27) | (t1 >>> 5); + bc1 = (t1 << 27) | (t0 >>> 5); + s[20] = bc0 ^ (bc4 & ~bc2); + s[21] = bc1 ^ (bc5 & ~bc3); + s[2] = bc2 ^ (bc6 & ~bc4); + s[3] = bc3 ^ (bc7 & ~bc5); + s[34] = bc4 ^ (bc8 & ~bc6); + s[35] = bc5 ^ (bc9 & ~bc7); + s[16] = bc6 ^ (bc0 & ~bc8); + s[17] = bc7 ^ (bc1 & ~bc9); + s[48] = bc8 ^ (bc2 & ~bc0); + s[49] = bc9 ^ (bc3 & ~bc1); + + t0 = s[10] ^ d0; + t1 = s[11] ^ d1; + bc6 = (t1 << 9) | (t0 >>> 23); + bc7 = (t0 << 9) | (t1 >>> 23); + t0 = s[42] ^ d2; + t1 = s[43] ^ d3; + bc8 = (t0 << 2) | (t1 >>> 30); + bc9 = (t1 << 2) | (t0 >>> 30); + t0 = s[24] ^ d4; + t1 = s[25] ^ d5; + bc0 = (t1 << 30) | (t0 >>> 2); + bc1 = (t0 << 30) | (t1 >>> 2); + t0 = s[6] ^ d6; + t1 = s[7] ^ d7; + bc2 = (t1 << 23) | (t0 >>> 9); + bc3 = (t0 << 23) | (t1 >>> 9); + t0 = s[38] ^ d8; + t1 = s[39] ^ d9; + bc4 = (t1 << 7) | (t0 >>> 25); + bc5 = (t0 << 7) | (t1 >>> 25); + s[10] = bc0 ^ (bc4 & ~bc2); + s[11] = bc1 ^ (bc5 & ~bc3); + s[42] = bc2 ^ (bc6 & ~bc4); + s[43] = bc3 ^ (bc7 & ~bc5); + s[24] = bc4 ^ (bc8 & ~bc6); + s[25] = bc5 ^ (bc9 & ~bc7); + s[6] = bc6 ^ (bc0 & ~bc8); + s[7] = bc7 ^ (bc1 & ~bc9); + s[38] = bc8 ^ (bc2 & ~bc0); + s[39] = bc9 ^ (bc3 & ~bc1); + + // Round 3 + bc0 = s[0] ^ s[10] ^ s[20] ^ s[30] ^ s[40]; + bc1 = s[1] ^ s[11] ^ s[21] ^ s[31] ^ s[41]; + bc2 = s[2] ^ s[12] ^ s[22] ^ s[32] ^ s[42]; + bc3 = s[3] ^ s[13] ^ s[23] ^ s[33] ^ s[43]; + bc4 = s[4] ^ s[14] ^ s[24] ^ s[34] ^ s[44]; + bc5 = s[5] ^ s[15] ^ s[25] ^ s[35] ^ s[45]; + bc6 = s[6] ^ s[16] ^ s[26] ^ s[36] ^ s[46]; + bc7 = s[7] ^ s[17] ^ s[27] ^ s[37] ^ s[47]; + bc8 = s[8] ^ s[18] ^ s[28] ^ s[38] ^ s[48]; + bc9 = s[9] ^ s[19] ^ s[29] ^ s[39] ^ s[49]; + + d0 = bc8 ^ ((bc2 << 1) | (bc3 >>> 31)); + d1 = bc9 ^ ((bc3 << 1) | (bc2 >>> 31)); + d2 = bc0 ^ ((bc4 << 1) | (bc5 >>> 31)); + d3 = bc1 ^ ((bc5 << 1) | (bc4 >>> 31)); + d4 = bc2 ^ ((bc6 << 1) | (bc7 >>> 31)); + d5 = bc3 ^ ((bc7 << 1) | (bc6 >>> 31)); + d6 = bc4 ^ ((bc8 << 1) | (bc9 >>> 31)); + d7 = bc5 ^ ((bc9 << 1) | (bc8 >>> 31)); + d8 = bc6 ^ ((bc0 << 1) | (bc1 >>> 31)); + d9 = bc7 ^ ((bc1 << 1) | (bc0 >>> 31)); + + bc0 = s[0] ^ d0; + bc1 = s[1] ^ d1; + t0 = s[22] ^ d2; + t1 = s[23] ^ d3; + bc2 = (t1 << 12) | (t0 >>> 20); + bc3 = (t0 << 12) | (t1 >>> 20); + t0 = s[44] ^ d4; + t1 = s[45] ^ d5; + bc4 = (t1 << 11) | (t0 >>> 21); + bc5 = (t0 << 11) | (t1 >>> 21); + t0 = s[16] ^ d6; + t1 = s[17] ^ d7; + bc6 = (t0 << 21) | (t1 >>> 11); + bc7 = (t1 << 21) | (t0 >>> 11); + t0 = s[38] ^ d8; + t1 = s[39] ^ d9; + bc8 = (t0 << 14) | (t1 >>> 18); + bc9 = (t1 << 14) | (t0 >>> 18); + s[0] = bc0 ^ (bc4 & ~bc2) ^ KECCAK_RC[n + 4]; + s[1] = bc1 ^ (bc5 & ~bc3) ^ KECCAK_RC[n + 5]; + s[22] = bc2 ^ (bc6 & ~bc4); + s[23] = bc3 ^ (bc7 & ~bc5); + s[44] = bc4 ^ (bc8 & ~bc6); + s[45] = bc5 ^ (bc9 & ~bc7); + s[16] = bc6 ^ (bc0 & ~bc8); + s[17] = bc7 ^ (bc1 & ~bc9); + s[38] = bc8 ^ (bc2 & ~bc0); + s[39] = bc9 ^ (bc3 & ~bc1); + + t0 = s[30] ^ d0; + t1 = s[31] ^ d1; + bc4 = (t0 << 3) | (t1 >>> 29); + bc5 = (t1 << 3) | (t0 >>> 29); + t0 = s[2] ^ d2; + t1 = s[3] ^ d3; + bc6 = (t1 << 13) | (t0 >>> 19); + bc7 = (t0 << 13) | (t1 >>> 19); + t0 = s[24] ^ d4; + t1 = s[25] ^ d5; + bc8 = (t1 << 29) | (t0 >>> 3); + bc9 = (t0 << 29) | (t1 >>> 3); + t0 = s[46] ^ d6; + t1 = s[47] ^ d7; + bc0 = (t0 << 28) | (t1 >>> 4); + bc1 = (t1 << 28) | (t0 >>> 4); + t0 = s[18] ^ d8; + t1 = s[19] ^ d9; + bc2 = (t0 << 20) | (t1 >>> 12); + bc3 = (t1 << 20) | (t0 >>> 12); + s[30] = bc0 ^ (bc4 & ~bc2); + s[31] = bc1 ^ (bc5 & ~bc3); + s[2] = bc2 ^ (bc6 & ~bc4); + s[3] = bc3 ^ (bc7 & ~bc5); + s[24] = bc4 ^ (bc8 & ~bc6); + s[25] = bc5 ^ (bc9 & ~bc7); + s[46] = bc6 ^ (bc0 & ~bc8); + s[47] = bc7 ^ (bc1 & ~bc9); + s[18] = bc8 ^ (bc2 & ~bc0); + s[19] = bc9 ^ (bc3 & ~bc1); + + t0 = s[10] ^ d0; + t1 = s[11] ^ d1; + bc8 = (t0 << 18) | (t1 >>> 14); + bc9 = (t1 << 18) | (t0 >>> 14); + t0 = s[32] ^ d2; + t1 = s[33] ^ d3; + bc0 = (t0 << 1) | (t1 >>> 31); + bc1 = (t1 << 1) | (t0 >>> 31); + t0 = s[4] ^ d4; + t1 = s[5] ^ d5; + bc2 = (t0 << 6) | (t1 >>> 26); + bc3 = (t1 << 6) | (t0 >>> 26); + t0 = s[26] ^ d6; + t1 = s[27] ^ d7; + bc4 = (t0 << 25) | (t1 >>> 7); + bc5 = (t1 << 25) | (t0 >>> 7); + t0 = s[48] ^ d8; + t1 = s[49] ^ d9; + bc6 = (t0 << 8) | (t1 >>> 24); + bc7 = (t1 << 8) | (t0 >>> 24); + s[10] = bc0 ^ (bc4 & ~bc2); + s[11] = bc1 ^ (bc5 & ~bc3); + s[32] = bc2 ^ (bc6 & ~bc4); + s[33] = bc3 ^ (bc7 & ~bc5); + s[4] = bc4 ^ (bc8 & ~bc6); + s[5] = bc5 ^ (bc9 & ~bc7); + s[26] = bc6 ^ (bc0 & ~bc8); + s[27] = bc7 ^ (bc1 & ~bc9); + s[48] = bc8 ^ (bc2 & ~bc0); + s[49] = bc9 ^ (bc3 & ~bc1); + + t0 = s[40] ^ d0; + t1 = s[41] ^ d1; + bc2 = (t1 << 4) | (t0 >>> 28); + bc3 = (t0 << 4) | (t1 >>> 28); + t0 = s[12] ^ d2; + t1 = s[13] ^ d3; + bc4 = (t0 << 10) | (t1 >>> 22); + bc5 = (t1 << 10) | (t0 >>> 22); + t0 = s[34] ^ d4; + t1 = s[35] ^ d5; + bc6 = (t0 << 15) | (t1 >>> 17); + bc7 = (t1 << 15) | (t0 >>> 17); + t0 = s[6] ^ d6; + t1 = s[7] ^ d7; + bc8 = (t1 << 24) | (t0 >>> 8); + bc9 = (t0 << 24) | (t1 >>> 8); + t0 = s[28] ^ d8; + t1 = s[29] ^ d9; + bc0 = (t0 << 27) | (t1 >>> 5); + bc1 = (t1 << 27) | (t0 >>> 5); + s[40] = bc0 ^ (bc4 & ~bc2); + s[41] = bc1 ^ (bc5 & ~bc3); + s[12] = bc2 ^ (bc6 & ~bc4); + s[13] = bc3 ^ (bc7 & ~bc5); + s[34] = bc4 ^ (bc8 & ~bc6); + s[35] = bc5 ^ (bc9 & ~bc7); + s[6] = bc6 ^ (bc0 & ~bc8); + s[7] = bc7 ^ (bc1 & ~bc9); + s[28] = bc8 ^ (bc2 & ~bc0); + s[29] = bc9 ^ (bc3 & ~bc1); + + t0 = s[20] ^ d0; + t1 = s[21] ^ d1; + bc6 = (t1 << 9) | (t0 >>> 23); + bc7 = (t0 << 9) | (t1 >>> 23); + t0 = s[42] ^ d2; + t1 = s[43] ^ d3; + bc8 = (t0 << 2) | (t1 >>> 30); + bc9 = (t1 << 2) | (t0 >>> 30); + t0 = s[14] ^ d4; + t1 = s[15] ^ d5; + bc0 = (t1 << 30) | (t0 >>> 2); + bc1 = (t0 << 30) | (t1 >>> 2); + t0 = s[36] ^ d6; + t1 = s[37] ^ d7; + bc2 = (t1 << 23) | (t0 >>> 9); + bc3 = (t0 << 23) | (t1 >>> 9); + t0 = s[8] ^ d8; + t1 = s[9] ^ d9; + bc4 = (t1 << 7) | (t0 >>> 25); + bc5 = (t0 << 7) | (t1 >>> 25); + s[20] = bc0 ^ (bc4 & ~bc2); + s[21] = bc1 ^ (bc5 & ~bc3); + s[42] = bc2 ^ (bc6 & ~bc4); + s[43] = bc3 ^ (bc7 & ~bc5); + s[14] = bc4 ^ (bc8 & ~bc6); + s[15] = bc5 ^ (bc9 & ~bc7); + s[36] = bc6 ^ (bc0 & ~bc8); + s[37] = bc7 ^ (bc1 & ~bc9); + s[8] = bc8 ^ (bc2 & ~bc0); + s[9] = bc9 ^ (bc3 & ~bc1); + + // Round 4 + bc0 = s[0] ^ s[10] ^ s[20] ^ s[30] ^ s[40]; + bc1 = s[1] ^ s[11] ^ s[21] ^ s[31] ^ s[41]; + bc2 = s[2] ^ s[12] ^ s[22] ^ s[32] ^ s[42]; + bc3 = s[3] ^ s[13] ^ s[23] ^ s[33] ^ s[43]; + bc4 = s[4] ^ s[14] ^ s[24] ^ s[34] ^ s[44]; + bc5 = s[5] ^ s[15] ^ s[25] ^ s[35] ^ s[45]; + bc6 = s[6] ^ s[16] ^ s[26] ^ s[36] ^ s[46]; + bc7 = s[7] ^ s[17] ^ s[27] ^ s[37] ^ s[47]; + bc8 = s[8] ^ s[18] ^ s[28] ^ s[38] ^ s[48]; + bc9 = s[9] ^ s[19] ^ s[29] ^ s[39] ^ s[49]; + + d0 = bc8 ^ ((bc2 << 1) | (bc3 >>> 31)); + d1 = bc9 ^ ((bc3 << 1) | (bc2 >>> 31)); + d2 = bc0 ^ ((bc4 << 1) | (bc5 >>> 31)); + d3 = bc1 ^ ((bc5 << 1) | (bc4 >>> 31)); + d4 = bc2 ^ ((bc6 << 1) | (bc7 >>> 31)); + d5 = bc3 ^ ((bc7 << 1) | (bc6 >>> 31)); + d6 = bc4 ^ ((bc8 << 1) | (bc9 >>> 31)); + d7 = bc5 ^ ((bc9 << 1) | (bc8 >>> 31)); + d8 = bc6 ^ ((bc0 << 1) | (bc1 >>> 31)); + d9 = bc7 ^ ((bc1 << 1) | (bc0 >>> 31)); + + bc0 = s[0] ^ d0; + bc1 = s[1] ^ d1; + t0 = s[2] ^ d2; + t1 = s[3] ^ d3; + bc2 = (t1 << 12) | (t0 >>> 20); + bc3 = (t0 << 12) | (t1 >>> 20); + t0 = s[4] ^ d4; + t1 = s[5] ^ d5; + bc4 = (t1 << 11) | (t0 >>> 21); + bc5 = (t0 << 11) | (t1 >>> 21); + t0 = s[6] ^ d6; + t1 = s[7] ^ d7; + bc6 = (t0 << 21) | (t1 >>> 11); + bc7 = (t1 << 21) | (t0 >>> 11); + t0 = s[8] ^ d8; + t1 = s[9] ^ d9; + bc8 = (t0 << 14) | (t1 >>> 18); + bc9 = (t1 << 14) | (t0 >>> 18); + s[0] = bc0 ^ (bc4 & ~bc2) ^ KECCAK_RC[n + 6]; + s[1] = bc1 ^ (bc5 & ~bc3) ^ KECCAK_RC[n + 7]; + s[2] = bc2 ^ (bc6 & ~bc4); + s[3] = bc3 ^ (bc7 & ~bc5); + s[4] = bc4 ^ (bc8 & ~bc6); + s[5] = bc5 ^ (bc9 & ~bc7); + s[6] = bc6 ^ (bc0 & ~bc8); + s[7] = bc7 ^ (bc1 & ~bc9); + s[8] = bc8 ^ (bc2 & ~bc0); + s[9] = bc9 ^ (bc3 & ~bc1); + + t0 = s[10] ^ d0; + t1 = s[11] ^ d1; + bc4 = (t0 << 3) | (t1 >>> 29); + bc5 = (t1 << 3) | (t0 >>> 29); + t0 = s[12] ^ d2; + t1 = s[13] ^ d3; + bc6 = (t1 << 13) | (t0 >>> 19); + bc7 = (t0 << 13) | (t1 >>> 19); + t0 = s[14] ^ d4; + t1 = s[15] ^ d5; + bc8 = (t1 << 29) | (t0 >>> 3); + bc9 = (t0 << 29) | (t1 >>> 3); + t0 = s[16] ^ d6; + t1 = s[17] ^ d7; + bc0 = (t0 << 28) | (t1 >>> 4); + bc1 = (t1 << 28) | (t0 >>> 4); + t0 = s[18] ^ d8; + t1 = s[19] ^ d9; + bc2 = (t0 << 20) | (t1 >>> 12); + bc3 = (t1 << 20) | (t0 >>> 12); + s[10] = bc0 ^ (bc4 & ~bc2); + s[11] = bc1 ^ (bc5 & ~bc3); + s[12] = bc2 ^ (bc6 & ~bc4); + s[13] = bc3 ^ (bc7 & ~bc5); + s[14] = bc4 ^ (bc8 & ~bc6); + s[15] = bc5 ^ (bc9 & ~bc7); + s[16] = bc6 ^ (bc0 & ~bc8); + s[17] = bc7 ^ (bc1 & ~bc9); + s[18] = bc8 ^ (bc2 & ~bc0); + s[19] = bc9 ^ (bc3 & ~bc1); + + t0 = s[20] ^ d0; + t1 = s[21] ^ d1; + bc8 = (t0 << 18) | (t1 >>> 14); + bc9 = (t1 << 18) | (t0 >>> 14); + t0 = s[22] ^ d2; + t1 = s[23] ^ d3; + bc0 = (t0 << 1) | (t1 >>> 31); + bc1 = (t1 << 1) | (t0 >>> 31); + t0 = s[24] ^ d4; + t1 = s[25] ^ d5; + bc2 = (t0 << 6) | (t1 >>> 26); + bc3 = (t1 << 6) | (t0 >>> 26); + t0 = s[26] ^ d6; + t1 = s[27] ^ d7; + bc4 = (t0 << 25) | (t1 >>> 7); + bc5 = (t1 << 25) | (t0 >>> 7); + t0 = s[28] ^ d8; + t1 = s[29] ^ d9; + bc6 = (t0 << 8) | (t1 >>> 24); + bc7 = (t1 << 8) | (t0 >>> 24); + s[20] = bc0 ^ (bc4 & ~bc2); + s[21] = bc1 ^ (bc5 & ~bc3); + s[22] = bc2 ^ (bc6 & ~bc4); + s[23] = bc3 ^ (bc7 & ~bc5); + s[24] = bc4 ^ (bc8 & ~bc6); + s[25] = bc5 ^ (bc9 & ~bc7); + s[26] = bc6 ^ (bc0 & ~bc8); + s[27] = bc7 ^ (bc1 & ~bc9); + s[28] = bc8 ^ (bc2 & ~bc0); + s[29] = bc9 ^ (bc3 & ~bc1); + + t0 = s[30] ^ d0; + t1 = s[31] ^ d1; + bc2 = (t1 << 4) | (t0 >>> 28); + bc3 = (t0 << 4) | (t1 >>> 28); + t0 = s[32] ^ d2; + t1 = s[33] ^ d3; + bc4 = (t0 << 10) | (t1 >>> 22); + bc5 = (t1 << 10) | (t0 >>> 22); + t0 = s[34] ^ d4; + t1 = s[35] ^ d5; + bc6 = (t0 << 15) | (t1 >>> 17); + bc7 = (t1 << 15) | (t0 >>> 17); + t0 = s[36] ^ d6; + t1 = s[37] ^ d7; + bc8 = (t1 << 24) | (t0 >>> 8); + bc9 = (t0 << 24) | (t1 >>> 8); + t0 = s[38] ^ d8; + t1 = s[39] ^ d9; + bc0 = (t0 << 27) | (t1 >>> 5); + bc1 = (t1 << 27) | (t0 >>> 5); + s[30] = bc0 ^ (bc4 & ~bc2); + s[31] = bc1 ^ (bc5 & ~bc3); + s[32] = bc2 ^ (bc6 & ~bc4); + s[33] = bc3 ^ (bc7 & ~bc5); + s[34] = bc4 ^ (bc8 & ~bc6); + s[35] = bc5 ^ (bc9 & ~bc7); + s[36] = bc6 ^ (bc0 & ~bc8); + s[37] = bc7 ^ (bc1 & ~bc9); + s[38] = bc8 ^ (bc2 & ~bc0); + s[39] = bc9 ^ (bc3 & ~bc1); + + t0 = s[40] ^ d0; + t1 = s[41] ^ d1; + bc6 = (t1 << 9) | (t0 >>> 23); + bc7 = (t0 << 9) | (t1 >>> 23); + t0 = s[42] ^ d2; + t1 = s[43] ^ d3; + bc8 = (t0 << 2) | (t1 >>> 30); + bc9 = (t1 << 2) | (t0 >>> 30); + t0 = s[44] ^ d4; + t1 = s[45] ^ d5; + bc0 = (t1 << 30) | (t0 >>> 2); + bc1 = (t0 << 30) | (t1 >>> 2); + t0 = s[46] ^ d6; + t1 = s[47] ^ d7; + bc2 = (t1 << 23) | (t0 >>> 9); + bc3 = (t0 << 23) | (t1 >>> 9); + t0 = s[48] ^ d8; + t1 = s[49] ^ d9; + bc4 = (t1 << 7) | (t0 >>> 25); + bc5 = (t0 << 7) | (t1 >>> 25); + s[40] = bc0 ^ (bc4 & ~bc2); + s[41] = bc1 ^ (bc5 & ~bc3); + s[42] = bc2 ^ (bc6 & ~bc4); + s[43] = bc3 ^ (bc7 & ~bc5); + s[44] = bc4 ^ (bc8 & ~bc6); + s[45] = bc5 ^ (bc9 & ~bc7); + s[46] = bc6 ^ (bc0 & ~bc8); + s[47] = bc7 ^ (bc1 & ~bc9); + s[48] = bc8 ^ (bc2 & ~bc0); + s[49] = bc9 ^ (bc3 & ~bc1); + } +} diff --git a/std/hash/_sha3/sha3.ts b/std/hash/_sha3/sha3.ts new file mode 100644 index 00000000000000..393889210bf0e9 --- /dev/null +++ b/std/hash/_sha3/sha3.ts @@ -0,0 +1,54 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +import { Sponge } from "./sponge.ts"; +import { keccakf } from "./keccakf.ts"; + +/* eslint-disable @typescript-eslint/camelcase, @typescript-eslint/class-name-casing */ + +/** Sha3-224 hash */ +export class Sha3_224 extends Sponge { + constructor() { + super({ + bitsize: 224, + rate: 144, + dsbyte: 6, + permutator: keccakf, + }); + } +} + +/** Sha3-256 hash */ +export class Sha3_256 extends Sponge { + constructor() { + super({ + bitsize: 256, + rate: 136, + dsbyte: 6, + permutator: keccakf, + }); + } +} + +/** Sha3-384 hash */ +export class Sha3_384 extends Sponge { + constructor() { + super({ + bitsize: 384, + rate: 104, + dsbyte: 6, + permutator: keccakf, + }); + } +} + +/** Sha3-512 hash */ +export class Sha3_512 extends Sponge { + constructor() { + super({ + bitsize: 512, + rate: 72, + dsbyte: 6, + permutator: keccakf, + }); + } +} diff --git a/std/hash/_sha3/shake.ts b/std/hash/_sha3/shake.ts new file mode 100644 index 00000000000000..05c699ea5ac1e0 --- /dev/null +++ b/std/hash/_sha3/shake.ts @@ -0,0 +1,51 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +import { Sponge } from "./sponge.ts"; +import { keccakf } from "./keccakf.ts"; + +/** Shake128 hash */ +export class Shake128 extends Sponge { + /** + * Instantiates a new Shake128 hash + * @param bitsize length of hash in bits + */ + constructor(bitsize: number) { + if (bitsize < 8) { + throw new Error("shake128: `bitsize` too small"); + } + + if (bitsize % 8 !== 0) { + throw new Error("shake128: `bitsize` must be multiple of 8"); + } + + super({ + bitsize: bitsize, + rate: 168, + dsbyte: 0x1f, + permutator: keccakf, + }); + } +} + +/** + * Instantiates a new Shake256 hash + * @param bitsize length of hash in bits + */ +export class Shake256 extends Sponge { + constructor(bitsize: number) { + if (bitsize < 8) { + throw new Error("shake256: `bitsize` too small"); + } + + if (bitsize % 8 !== 0) { + throw new Error("shake256: `bitsize` must be multiple of 8"); + } + + super({ + bitsize: bitsize, + rate: 136, + dsbyte: 0x1f, + permutator: keccakf, + }); + } +} diff --git a/std/hash/_sha3/sponge.ts b/std/hash/_sha3/sponge.ts new file mode 100644 index 00000000000000..a5705e13e231f0 --- /dev/null +++ b/std/hash/_sha3/sponge.ts @@ -0,0 +1,111 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +import * as hex from "../../encoding/hex.ts"; + +type SpongePermutator = (data: Uint8Array) => void; + +/** Sponge construction option */ +export interface SpongeOption { + bitsize: number; + rate: number; + dsbyte: number; + permutator: SpongePermutator; +} + +export type Message = string | ArrayBuffer; + +const STATE_SIZE = 200; +const TYPE_ERROR_MSG = "sha3: `data` is invalid type"; + +/** Sponge construction */ +export class Sponge { + #option: SpongeOption; + #state: Uint8Array; + #rp: number; + #absorbing: boolean; + + constructor(option: SpongeOption) { + this.#option = option; + this.#state = new Uint8Array(STATE_SIZE); + this.#rp = 0; + this.#absorbing = true; + } + + /** Applies padding to internal state */ + private pad(): void { + this.#state[this.#rp] ^= this.#option.dsbyte; + this.#state[this.#option.rate - 1] ^= 0x80; + } + + /** Squeezes internal state */ + protected squeeze(length: number): Uint8Array { + if (length < 0) { + throw new Error("sha3: length cannot be negative"); + } + + this.pad(); + + const hash = new Uint8Array(length); + let pos = 0; + while (length > 0) { + const r = length > this.#option.rate ? this.#option.rate : length; + this.#option.permutator(this.#state); + hash.set(this.#state.slice(0, r), pos); + length -= r; + pos += r; + } + + this.#absorbing = false; + return hash; + } + + /** Updates internal state by absorbing */ + update(data: Message): this { + if (!this.#absorbing) { + throw new Error("sha3: cannot update already finalized hash"); + } + + let msg: Uint8Array; + + if (typeof data === "string") { + msg = new TextEncoder().encode(data as string); + } else if (typeof data === "object") { + if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) { + msg = new Uint8Array(data); + } else { + throw new Error(TYPE_ERROR_MSG); + } + } else { + throw new Error(TYPE_ERROR_MSG); + } + + let rp = this.#rp; + + for (let i = 0; i < msg.length; ++i) { + this.#state[rp++] ^= msg[i]; + if (rp >= this.#option.rate) { + this.#option.permutator(this.#state); + rp = 0; + } + } + + this.#rp = rp; + return this; + } + + /** Returns the hash in ArrayBuffer */ + digest(): ArrayBuffer { + return this.squeeze(this.#option.bitsize >> 3); + } + + /** Returns the hash in given format */ + toString(format: "hex" = "hex"): string { + const rawOutput = this.squeeze(this.#option.bitsize >> 3); + switch (format) { + case "hex": + return hex.encodeToString(rawOutput); + default: + throw new Error("sha3: invalid output format"); + } + } +} diff --git a/std/hash/sha1_test.ts b/std/hash/sha1_test.ts index 0387c914ae8de6..36702d55afb700 100644 --- a/std/hash/sha1_test.ts +++ b/std/hash/sha1_test.ts @@ -18,7 +18,7 @@ function toHexString(value: number[] | ArrayBuffer): string { } // prettier-ignore -// dprint-ignore +// deno-fmt-ignore const fixtures: { sha1: Record>; } = { diff --git a/std/hash/sha256.ts b/std/hash/sha256.ts index c5635cacd4ac4c..61da5a578ded26 100644 --- a/std/hash/sha256.ts +++ b/std/hash/sha256.ts @@ -15,7 +15,7 @@ const HEX_CHARS = "0123456789abcdef".split(""); const EXTRA = [-2147483648, 8388608, 32768, 128] as const; const SHIFT = [24, 16, 8, 0] as const; // prettier-ignore -// dprint-ignore +// deno-fmt-ignore const K = [ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, diff --git a/std/hash/sha256_test.ts b/std/hash/sha256_test.ts index 92c7e3d5b334b2..68c67b5ee24d3b 100644 --- a/std/hash/sha256_test.ts +++ b/std/hash/sha256_test.ts @@ -19,7 +19,7 @@ function toHexString(value: number[] | ArrayBuffer): string { } // prettier-ignore -// dprint-ignore +// deno-fmt-ignore const fixtures: { sha256: Record>; sha224: Record>; @@ -158,35 +158,35 @@ const fixtures: { }; // prettier-ignore -// dprint-ignore +// deno-fmt-ignore fixtures.sha256.Uint8Array = { '182889f925ae4e5cc37118ded6ed87f7bdc7cab5ec5e78faef2e50048999473f': new Uint8Array([211, 212]), 'd7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592': new Uint8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) }; // prettier-ignore -// dprint-ignore +// deno-fmt-ignore fixtures.sha256.Int8Array = { 'd7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592': new Int8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) }; // prettier-ignore -// dprint-ignore +// deno-fmt-ignore fixtures.sha256.ArrayBuffer = { 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855': new ArrayBuffer(0), '6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d': new ArrayBuffer(1) }; // prettier-ignore -// dprint-ignore +// deno-fmt-ignore fixtures.sha224.Uint8Array = { 'e17541396a3ecd1cd5a2b968b84e597e8eae3b0ea3127963bf48dd3b': new Uint8Array([211, 212]), '730e109bd7a8a32b1cb9d9a09aa2325d2430587ddbc0c38bad911525': new Uint8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) }; // prettier-ignore -// dprint-ignore +// deno-fmt-ignore fixtures.sha224.Int8Array = { '730e109bd7a8a32b1cb9d9a09aa2325d2430587ddbc0c38bad911525': new Int8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) }; // prettier-ignore -// dprint-ignore +// deno-fmt-ignore fixtures.sha224.ArrayBuffer = { 'd14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f': new ArrayBuffer(0), 'fff9292b4201617bdc4d3053fce02734166a683d7d858a7f5f59b073': new ArrayBuffer(1), diff --git a/std/hash/sha3.ts b/std/hash/sha3.ts new file mode 100644 index 00000000000000..b5154fbdff63f6 --- /dev/null +++ b/std/hash/sha3.ts @@ -0,0 +1,6 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +/* eslint-disable-next-line @typescript-eslint/camelcase */ +export { Sha3_224, Sha3_256, Sha3_384, Sha3_512 } from "./_sha3/sha3.ts"; +export { Keccak224, Keccak256, Keccak384, Keccak512 } from "./_sha3/keccak.ts"; +export { Shake128, Shake256 } from "./_sha3/shake.ts"; diff --git a/std/hash/sha3_test.ts b/std/hash/sha3_test.ts new file mode 100644 index 00000000000000..6812f6209e775a --- /dev/null +++ b/std/hash/sha3_test.ts @@ -0,0 +1,575 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +/* eslint-disable @typescript-eslint/camelcase */ + +const { test } = Deno; +import { assertEquals, assertThrows } from "../testing/asserts.ts"; +import { + Keccak224, + Keccak256, + Keccak384, + Keccak512, + Sha3_224, + Sha3_256, + Sha3_384, + Sha3_512, + Shake128, + Shake256, +} from "./sha3.ts"; +import * as hex from "../encoding/hex.ts"; + +const millionAs = "a".repeat(1000000); + +const testSetSha3_224 = [ + ["", "6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7"], + ["abc", "e642824c3f8cf24ad09234ee7d3c766fc9a3a5168d0c94ad73b46fdf"], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "8a24108b154ada21c9fd5574494479ba5c7e7ab76ef264ead0fcce33", + ], + [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "f9019111996dcf160e284e320fd6d8825cabcd41a5ffdc4c5e9d64b6", + ], + [millionAs, "d69335b93325192e516a912e6d19a15cb51c6ed5c15243e7a7fd653c"], +]; + +const testSetSha3_256 = [ + ["", "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"], + ["abc", "3a985da74fe225b2045c172d6bd390bd855f086e3e9d525b46bfe24511431532"], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "41c0dba2a9d6240849100376a8235e2c82e1b9998a999e21db32dd97496d3376", + ], + [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "3fc5559f14db8e453a0a3091edbd2bc25e11528d81c66fa570a4efdcc2695ee1", + ], + [ + millionAs, + "5c8875ae474a3634ba4fd55ec85bffd661f32aca75c6d699d0cdcb6c115891c1", + ], +]; + +const testSetSha3_384 = [ + [ + "", + "0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2ac3713831264adb47fb6bd1e058d5f004", + ], + [ + "abc", + "ec01498288516fc926459f58e2c6ad8df9b473cb0fc08c2596da7cf0e49be4b298d88cea927ac7f539f1edf228376d25", + ], + [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "3a4f3b6284e571238884e95655e8c8a60e068e4059a9734abc08823a900d161592860243f00619ae699a29092ed91a16", + ], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "991c665755eb3a4b6bbdfb75c78a492e8c56a22c5c4d7e429bfdbc32b9d4ad5aa04a1f076e62fea19eef51acd0657c22", + ], + [ + millionAs, + "eee9e24d78c1855337983451df97c8ad9eedf256c6334f8e948d252d5e0e76847aa0774ddb90a842190d2c558b4b8340", + ], +]; + +const testSetSha3_512 = [ + [ + "", + "a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a615b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26", + ], + [ + "abc", + "b751850b1a57168a5693cd924b6b096e08f621827444f70d884f5d0240d2712e10e116e9192af3c91a7ec57647e3934057340b4cf408d5a56592f8274eec53f0", + ], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "04a371e84ecfb5b8b77cb48610fca8182dd457ce6f326a0fd3d7ec2f1e91636dee691fbe0c985302ba1b0d8dc78c086346b533b49c030d99a27daf1139d6e75e", + ], + [ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "a8ae722a78e10cbbc413886c02eb5b369a03f6560084aff566bd597bb7ad8c1ccd86e81296852359bf2faddb5153c0a7445722987875e74287adac21adebe952", + ], + [ + millionAs, + "3c3a876da14034ab60627c077bb98f7e120a2a5370212dffb3385a18d4f38859ed311d0a9d5141ce9cc5c66ee689b266a8aa18ace8282a0e0db596c90b0a7b87", + ], +]; + +const testSetKeccak224 = [ + ["", "f71837502ba8e10837bdd8d365adb85591895602fc552b48b7390abd"], + ["abc", "c30411768506ebe1c2871b1ee2e87d38df342317300a9b97a95ec6a8"], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "e51faa2b4655150b931ee8d700dc202f763ca5f962c529eae55012b6", + ], + [millionAs, "19f9167be2a04c43abd0ed554788101b9c339031acc8e1468531303f"], +]; + +const testSetKeccak256 = [ + ["", "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"], + ["abc", "4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45"], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "45d3b367a6904e6e8d502ee04999a7c27647f91fa845d456525fd352ae3d7371", + ], + [ + millionAs, + "fadae6b49f129bbb812be8407b7b2894f34aecf6dbd1f9b0f0c7e9853098fc96", + ], +]; + +const testSetKeccak384 = [ + [ + "", + "2c23146a63a29acf99e73b88f8c24eaa7dc60aa771780ccc006afbfa8fe2479b2dd2b21362337441ac12b515911957ff", + ], + [ + "abc", + "f7df1165f033337be098e7d288ad6a2f74409d7a60b49c36642218de161b1f99f8c681e4afaf31a34db29fb763e3c28e", + ], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "b41e8896428f1bcbb51e17abd6acc98052a3502e0d5bf7fa1af949b4d3c855e7c4dc2c390326b3f3e74c7b1e2b9a3657", + ], + [ + millionAs, + "0c8324e1ebc182822c5e2a086cac07c2fe00e3bce61d01ba8ad6b71780e2dec5fb89e5ae90cb593e57bc6258fdd94e17", + ], +]; + +const testSetKeccak512 = [ + [ + "", + "0eab42de4c3ceb9235fc91acffe746b29c29a8c366b7c60e4e67c466f36a4304c00fa9caf9d87976ba469bcbe06713b435f091ef2769fb160cdab33d3670680e", + ], + [ + "abc", + "18587dc2ea106b9a1563e32b3312421ca164c7f1f07bc922a9c83d77cea3a1e5d0c69910739025372dc14ac9642629379540c17e2a65b19d77aa511a9d00bb96", + ], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "6aa6d3669597df6d5a007b00d09c20795b5c4218234e1698a944757a488ecdc09965435d97ca32c3cfed7201ff30e070cd947f1fc12b9d9214c467d342bcba5d", + ], + [ + millionAs, + "5cf53f2e556be5a624425ede23d0e8b2c7814b4ba0e4e09cbbf3c2fac7056f61e048fc341262875ebc58a5183fea651447124370c1ebf4d6c89bc9a7731063bb", + ], +]; + +const testSetShake128 = [ + ["", "7f9c2ba4e88f827d616045507605853e"], + ["abc", "5881092dd818bf5cf8a3ddb793fbcba7"], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "1a96182b50fb8c7e74e0a707788f55e9", + ], + [millionAs, "9d222c79c4ff9d092cf6ca86143aa411"], +]; + +const testSetShake128_224 = [ + ["", "7f9c2ba4e88f827d616045507605853ed73b8093f6efbc88eb1a6eac"], + ["abc", "5881092dd818bf5cf8a3ddb793fbcba74097d5c526a6d35f97b83351"], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "1a96182b50fb8c7e74e0a707788f55e98209b8d91fade8f32f8dd5cf", + ], + [millionAs, "9d222c79c4ff9d092cf6ca86143aa411e369973808ef97093255826c"], +]; + +const testSetShake128_2048 = [ + [ + "", + "7f9c2ba4e88f827d616045507605853ed73b8093f6efbc88eb1a6eacfa66ef263cb1eea988004b93103cfb0aeefd2a686e01fa4a58e8a3639ca8a1e3f9ae57e235b8cc873c23dc62b8d260169afa2f75ab916a58d974918835d25e6a435085b2badfd6dfaac359a5efbb7bcc4b59d538df9a04302e10c8bc1cbf1a0b3a5120ea17cda7cfad765f5623474d368ccca8af0007cd9f5e4c849f167a580b14aabdefaee7eef47cb0fca9767be1fda69419dfb927e9df07348b196691abaeb580b32def58538b8d23f87732ea63b02b4fa0f4873360e2841928cd60dd4cee8cc0d4c922a96188d032675c8ac850933c7aff1533b94c834adbb69c6115bad4692d8619", + ], + [ + "abc", + "5881092dd818bf5cf8a3ddb793fbcba74097d5c526a6d35f97b83351940f2cc844c50af32acd3f2cdd066568706f509bc1bdde58295dae3f891a9a0fca5783789a41f8611214ce612394df286a62d1a2252aa94db9c538956c717dc2bed4f232a0294c857c730aa16067ac1062f1201fb0d377cfb9cde4c63599b27f3462bba4a0ed296c801f9ff7f57302bb3076ee145f97a32ae68e76ab66c48d51675bd49acc29082f5647584e6aa01b3f5af057805f973ff8ecb8b226ac32ada6f01c1fcd4818cb006aa5b4cdb3611eb1e533c8964cacfdf31012cd3fb744d02225b988b475375faad996eb1b9176ecb0f8b2871723d6dbb804e23357e50732f5cfc904b1", + ], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "1a96182b50fb8c7e74e0a707788f55e98209b8d91fade8f32f8dd5cff7bf21f54ee5f19550825a6e070030519e944263ac1c6765287065621f9fcb3201723e3223b63a46c2938aa953ba8401d0ea77b8d26490775566407b95673c0f4cc1ce9fd966148d7efdff26bbf9f48a21c6da35bfaa545654f70ae586ff10131420771483ec92edab408c767bf4c5b4fffaa80c8ca214d84c4dc700d0c50630b2ffc3793ea4d87258b4c9548c5485a5ca666ef73fbd816d418aea6395b503addd9b150f9e0663325f01e5518b71ffa1244ea284cebe0cea2f774d7b3a437dca3282e324777e19624bf2be3cd355c1bfbddb323a33f11efafb2448293501dc0454c6b72f", + ], + [ + millionAs, + "9d222c79c4ff9d092cf6ca86143aa411e369973808ef97093255826c5572ef58424c4b5c28475ffdcf981663867fec6321c1262e387bccf8ca676884c4a9d0c13bfa6869763d5ae4bbc9b3ccd09d1ca5ea7446538d69b3fb98c72b59a2b4817db5eadd9011f90fa71091931f8134f4f00b562e2fe105937270361c1909862ad45046e3932f5dd311ec72fec5f8fb8f60b45a3bee3f85bbf7fcedc6a555677648e0654b381941a86bd3e512657b0d57a7991fc4543f89d8290492222ce4a33e17602b3b99c009f7655f87535cdaa3716f58c47b8a157ad195f02809f27500b9254979311c6bb415968cd10431169a27d5a8d61e13a6b8b77af1f8b6dd2eefdea0", + ], +]; + +const testSetShake256 = [ + ["", "46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762f"], + ["abc", "483366601360a8771c6863080cc4114d8db44530f8f1e1ee4f94ea37e78b5739"], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "4d8c2dd2435a0128eefbb8c36f6f87133a7911e18d979ee1ae6be5d4fd2e3329", + ], + [ + millionAs, + "3578a7a4ca9137569cdf76ed617d31bb994fca9c1bbf8b184013de8234dfd13a", + ], +]; + +const testSetShake256_128 = [ + ["", "46b9dd2b0ba88d13233b3feb743eeb24"], + ["abc", "483366601360a8771c6863080cc4114d"], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "4d8c2dd2435a0128eefbb8c36f6f8713", + ], + [millionAs, "3578a7a4ca9137569cdf76ed617d31bb"], +]; + +const testSetShake256_384 = [ + [ + "", + "46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762fd75dc4ddd8c0f200cb05019d67b592f6", + ], + [ + "abc", + "483366601360a8771c6863080cc4114d8db44530f8f1e1ee4f94ea37e78b5739d5a15bef186a5386c75744c0527e1faa", + ], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "4d8c2dd2435a0128eefbb8c36f6f87133a7911e18d979ee1ae6be5d4fd2e332940d8688a4e6a59aa8060f1f9bc996c05", + ], + [ + millionAs, + "3578a7a4ca9137569cdf76ed617d31bb994fca9c1bbf8b184013de8234dfd13a3fd124d4df76c0a539ee7dd2f6e1ec34", + ], +]; + +const testSetShake256_512 = [ + [ + "", + "46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762fd75dc4ddd8c0f200cb05019d67b592f6fc821c49479ab48640292eacb3b7c4be", + ], + [ + "abc", + "483366601360a8771c6863080cc4114d8db44530f8f1e1ee4f94ea37e78b5739d5a15bef186a5386c75744c0527e1faa9f8726e462a12a4feb06bd8801e751e4", + ], + [ + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "4d8c2dd2435a0128eefbb8c36f6f87133a7911e18d979ee1ae6be5d4fd2e332940d8688a4e6a59aa8060f1f9bc996c05aca3c696a8b66279dc672c740bb224ec", + ], + [ + millionAs, + "3578a7a4ca9137569cdf76ed617d31bb994fca9c1bbf8b184013de8234dfd13a3fd124d4df76c0a539ee7dd2f6e1ec346124c815d9410e145eb561bcd97b18ab", + ], +]; + +function s2b(data: string): Uint8Array { + return new TextEncoder().encode(data); +} + +test("[hash/sha3] testSha3-224Raw", () => { + const sha3sum = (data: ArrayBuffer): ArrayBuffer => { + const sha3 = new Sha3_224(); + return sha3.update(data).digest(); + }; + + for (const [input, output] of testSetSha3_224) { + const rawOutput = hex.decodeString(output); + assertEquals(sha3sum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testSha3-224String", () => { + const sha3sum = (data: string): string => { + const sha3 = new Sha3_224(); + return sha3.update(data).toString(); + }; + + for (const [input, output] of testSetSha3_224) { + assertEquals(sha3sum(input), output); + } +}); + +test("[hash/sha3] testSha3-256Raw", () => { + const sha3sum = (data: ArrayBuffer): ArrayBuffer => { + const sha3 = new Sha3_256(); + return sha3.update(data).digest(); + }; + + for (const [input, output] of testSetSha3_256) { + const rawOutput = hex.decodeString(output); + assertEquals(sha3sum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testSha3-256String", () => { + const sha3sum = (data: string): string => { + const sha3 = new Sha3_256(); + return sha3.update(data).toString(); + }; + + for (const [input, output] of testSetSha3_256) { + assertEquals(sha3sum(input), output); + } +}); + +test("[hash/sha3] testSha3-384Raw", () => { + const sha3sum = (data: ArrayBuffer): ArrayBuffer => { + const sha3 = new Sha3_384(); + return sha3.update(data).digest(); + }; + + for (const [input, output] of testSetSha3_384) { + const rawOutput = hex.decodeString(output); + assertEquals(sha3sum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testSha3-384String", () => { + const sha3sum = (data: string): string => { + const sha3 = new Sha3_384(); + return sha3.update(data).toString(); + }; + + for (const [input, output] of testSetSha3_384) { + assertEquals(sha3sum(input), output); + } +}); + +test("[hash/sha3] testSha3-512Raw", () => { + const sha3sum = (data: ArrayBuffer): ArrayBuffer => { + const sha3 = new Sha3_512(); + return sha3.update(data).digest(); + }; + + for (const [input, output] of testSetSha3_512) { + const rawOutput = hex.decodeString(output); + assertEquals(sha3sum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testSha3-512String", () => { + const sha3sum = (data: string): string => { + const sha3 = new Sha3_512(); + return sha3.update(data).toString(); + }; + + for (const [input, output] of testSetSha3_512) { + assertEquals(sha3sum(input), output); + } +}); + +test("[hash/sha3] testKeccak-224Raw", () => { + const keccakSum = (data: ArrayBuffer): ArrayBuffer => { + const keccak = new Keccak224(); + return keccak.update(data).digest(); + }; + + for (const [input, output] of testSetKeccak224) { + const rawOutput = hex.decodeString(output); + assertEquals(keccakSum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testKeccak-224String", () => { + const keccakSum = (data: string): string => { + const keccak = new Keccak224(); + return keccak.update(data).toString(); + }; + + for (const [input, output] of testSetKeccak224) { + assertEquals(keccakSum(input), output); + } +}); + +test("[hash/sha3] testKeccak-256Raw", () => { + const keccakSum = (data: ArrayBuffer): ArrayBuffer => { + const keccak = new Keccak256(); + return keccak.update(data).digest(); + }; + + for (const [input, output] of testSetKeccak256) { + const rawOutput = hex.decodeString(output); + assertEquals(keccakSum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testKeccak-256String", () => { + const keccakSum = (data: string): string => { + const keccak = new Keccak256(); + return keccak.update(data).toString(); + }; + + for (const [input, output] of testSetKeccak256) { + assertEquals(keccakSum(input), output); + } +}); + +test("[hash/sha3] testKeccak-384Raw", () => { + const keccakSum = (data: ArrayBuffer): ArrayBuffer => { + const keccak = new Keccak384(); + return keccak.update(data).digest(); + }; + + for (const [input, output] of testSetKeccak384) { + const rawOutput = hex.decodeString(output); + assertEquals(keccakSum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testKeccak-384String", () => { + const keccakSum = (data: string): string => { + const keccak = new Keccak384(); + return keccak.update(data).toString(); + }; + + for (const [input, output] of testSetKeccak384) { + assertEquals(keccakSum(input), output); + } +}); + +test("[hash/sha3] testKeccak-512Raw", () => { + const keccakSum = (data: ArrayBuffer): ArrayBuffer => { + const keccak = new Keccak512(); + return keccak.update(data).digest(); + }; + + for (const [input, output] of testSetKeccak512) { + const rawOutput = hex.decodeString(output); + assertEquals(keccakSum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testKeccak-512String", () => { + const keccakSum = (data: string): string => { + const keccak = new Keccak512(); + return keccak.update(data).toString(); + }; + + for (const [input, output] of testSetKeccak512) { + assertEquals(keccakSum(input), output); + } +}); + +test("[hash/sha3] testSHAKE-128Raw", () => { + const shakeSum = (data: ArrayBuffer): ArrayBuffer => { + const shake = new Shake128(128); + return shake.update(data).digest(); + }; + + for (const [input, output] of testSetShake128) { + const rawOutput = hex.decodeString(output); + assertEquals(shakeSum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testSHAKE-128String", () => { + const shakeSum = (data: string): string => { + const shake = new Shake128(128); + return shake.update(data).toString(); + }; + + for (const [input, output] of testSetShake128) { + assertEquals(shakeSum(input), output); + } +}); + +test("[hash/sha3] testSHAKE-128-224Raw", () => { + const shakeSum = (data: ArrayBuffer): ArrayBuffer => { + const shake = new Shake128(224); + return shake.update(data).digest(); + }; + + for (const [input, output] of testSetShake128_224) { + const rawOutput = hex.decodeString(output); + assertEquals(shakeSum(s2b(input)), rawOutput); + } +}); + +test("[hash/sha3] testSHAKE-128-224String", () => { + const shakeSum = (data: string): string => { + const shake = new Shake128(224); + return shake.update(data).toString(); + }; + + for (const [input, output] of testSetShake128_224) { + assertEquals(shakeSum(input), output); + } +}); + +test("[hash/sha3] testSHAKE-128-2048", () => { + const shakeSum = (data: string): string => { + const shake = new Shake128(2048); + return shake.update(data).toString(); + }; + + for (const [input, output] of testSetShake128_2048) { + assertEquals(shakeSum(input), output); + } +}); + +test("[hash/sha3] testSHAKE-256", () => { + const shakeSum = (data: string): string => { + const shake = new Shake256(256); + return shake.update(data).toString(); + }; + + for (const [input, output] of testSetShake256) { + assertEquals(shakeSum(input), output); + } +}); + +test("[hash/sha3] testSHAKE-256-128", () => { + const shakeSum = (data: string): string => { + const shake = new Shake256(128); + return shake.update(data).toString(); + }; + + for (const [input, output] of testSetShake256_128) { + assertEquals(shakeSum(input), output); + } +}); + +test("[hash/sha3] testSHAKE-256-384", () => { + const shakeSum = (data: string): string => { + const shake = new Shake256(384); + return shake.update(data).toString(); + }; + + for (const [input, output] of testSetShake256_384) { + assertEquals(shakeSum(input), output); + } +}); + +test("[hash/sha3] testSHAKE-256-512", () => { + const shakeSum = (data: string): string => { + const shake = new Shake256(512); + return shake.update(data).toString(); + }; + + for (const [input, output] of testSetShake256_512) { + assertEquals(shakeSum(input), output); + } +}); + +test("[hash/sha3] testSha3-256Chain", () => { + const sha3 = new Sha3_256(); + const output = sha3 + .update(s2b("a")) + .update(s2b("b")) + .update(s2b("c")) + .toString(); + + assertEquals( + output, + "3a985da74fe225b2045c172d6bd390bd855f086e3e9d525b46bfe24511431532" + ); +}); + +test("[hash/sha3] testSha3UpdateFinalized", () => { + assertThrows( + () => { + const sha3 = new Sha3_256(); + const hash = sha3.update(s2b("a")).digest(); + const hash2 = sha3.update(s2b("a")).digest(); + assertEquals(hash, hash2); + }, + Error, + "sha3: cannot update already finalized hash" + ); +}); diff --git a/std/hash/sha512.ts b/std/hash/sha512.ts new file mode 100644 index 00000000000000..b55069f4d7693e --- /dev/null +++ b/std/hash/sha512.ts @@ -0,0 +1,791 @@ +/* + * [js-sha512]{@link https://github.com/emn178/js-sha512} + * + * @version 0.8.0 + * @author Chen, Yi-Cyuan [emn178@gmail.com] + * @copyright Chen, Yi-Cyuan 2014-2018 + * @license MIT + */ + +export type Message = string | number[] | ArrayBuffer; + +// prettier-ignore +const HEX_CHARS = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"] as const; +const EXTRA = [-2147483648, 8388608, 32768, 128] as const; +const SHIFT = [24, 16, 8, 0] as const; +// prettier-ignore +const K = [ + 0x428a2f98, 0xd728ae22, 0x71374491, 0x23ef65cd, 0xb5c0fbcf, 0xec4d3b2f, 0xe9b5dba5, 0x8189dbbc, 0x3956c25b, + 0xf348b538, 0x59f111f1, 0xb605d019, 0x923f82a4, 0xaf194f9b, 0xab1c5ed5, 0xda6d8118, 0xd807aa98, 0xa3030242, + 0x12835b01, 0x45706fbe, 0x243185be, 0x4ee4b28c, 0x550c7dc3, 0xd5ffb4e2, 0x72be5d74, 0xf27b896f, 0x80deb1fe, + 0x3b1696b1, 0x9bdc06a7, 0x25c71235, 0xc19bf174, 0xcf692694, 0xe49b69c1, 0x9ef14ad2, 0xefbe4786, 0x384f25e3, + 0x0fc19dc6, 0x8b8cd5b5, 0x240ca1cc, 0x77ac9c65, 0x2de92c6f, 0x592b0275, 0x4a7484aa, 0x6ea6e483, 0x5cb0a9dc, + 0xbd41fbd4, 0x76f988da, 0x831153b5, 0x983e5152, 0xee66dfab, 0xa831c66d, 0x2db43210, 0xb00327c8, 0x98fb213f, + 0xbf597fc7, 0xbeef0ee4, 0xc6e00bf3, 0x3da88fc2, 0xd5a79147, 0x930aa725, 0x06ca6351, 0xe003826f, 0x14292967, + 0x0a0e6e70, 0x27b70a85, 0x46d22ffc, 0x2e1b2138, 0x5c26c926, 0x4d2c6dfc, 0x5ac42aed, 0x53380d13, 0x9d95b3df, + 0x650a7354, 0x8baf63de, 0x766a0abb, 0x3c77b2a8, 0x81c2c92e, 0x47edaee6, 0x92722c85, 0x1482353b, 0xa2bfe8a1, + 0x4cf10364, 0xa81a664b, 0xbc423001, 0xc24b8b70, 0xd0f89791, 0xc76c51a3, 0x0654be30, 0xd192e819, 0xd6ef5218, + 0xd6990624, 0x5565a910, 0xf40e3585, 0x5771202a, 0x106aa070, 0x32bbd1b8, 0x19a4c116, 0xb8d2d0c8, 0x1e376c08, + 0x5141ab53, 0x2748774c, 0xdf8eeb99, 0x34b0bcb5, 0xe19b48a8, 0x391c0cb3, 0xc5c95a63, 0x4ed8aa4a, 0xe3418acb, + 0x5b9cca4f, 0x7763e373, 0x682e6ff3, 0xd6b2b8a3, 0x748f82ee, 0x5defb2fc, 0x78a5636f, 0x43172f60, 0x84c87814, + 0xa1f0ab72, 0x8cc70208, 0x1a6439ec, 0x90befffa, 0x23631e28, 0xa4506ceb, 0xde82bde9, 0xbef9a3f7, 0xb2c67915, + 0xc67178f2, 0xe372532b, 0xca273ece, 0xea26619c, 0xd186b8c7, 0x21c0c207, 0xeada7dd6, 0xcde0eb1e, 0xf57d4f7f, + 0xee6ed178, 0x06f067aa, 0x72176fba, 0x0a637dc5, 0xa2c898a6, 0x113f9804, 0xbef90dae, 0x1b710b35, 0x131c471b, + 0x28db77f5, 0x23047d84, 0x32caab7b, 0x40c72493, 0x3c9ebe0a, 0x15c9bebc, 0x431d67c4, 0x9c100d4c, 0x4cc5d4be, + 0xcb3e42b6, 0x597f299c, 0xfc657e2a, 0x5fcb6fab, 0x3ad6faec, 0x6c44198c, 0x4a475817 +] as const; + +const blocks: number[] = []; + +// prettier-ignore +export class Sha512 { + #blocks!: number[]; + #block!: number; + #bits!: number; + #start!: number; + #bytes!: number; + #hBytes!: number; + #lastByteIndex = 0; + #finalized!: boolean; + #hashed!: boolean; + #h0h!: number; + #h0l!: number; + #h1h!: number; + #h1l!: number; + #h2h!: number; + #h2l!: number; + #h3h!: number; + #h3l!: number; + #h4h!: number; + #h4l!: number; + #h5h!: number; + #h5l!: number; + #h6h!: number; + #h6l!: number; + #h7h!: number; + #h7l!: number; + + constructor(bits = 512, sharedMemory = false) { + this.init(bits, sharedMemory); + } + + protected init(bits: number, sharedMemory: boolean): void { + if (sharedMemory) { + blocks[0] = blocks[1] = blocks[2] = blocks[3] = blocks[4] = blocks[5] = blocks[6] = blocks[7] = blocks[8] = + blocks[9] = blocks[10] = blocks[11] = blocks[12] = blocks[13] = blocks[14] = blocks[15] = blocks[16] = + blocks[17] = blocks[18] = blocks[19] = blocks[20] = blocks[21] = blocks[22] = blocks[23] = blocks[24] = + blocks[25] = blocks[26] = blocks[27] = blocks[28] = blocks[29] = blocks[30] = blocks[31] = blocks[32] = 0; + this.#blocks = blocks; + } else { + this.#blocks = + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + } + if (bits === 224) { + this.#h0h = 0x8c3d37c8; + this.#h0l = 0x19544da2; + this.#h1h = 0x73e19966; + this.#h1l = 0x89dcd4d6; + this.#h2h = 0x1dfab7ae; + this.#h2l = 0x32ff9c82; + this.#h3h = 0x679dd514; + this.#h3l = 0x582f9fcf; + this.#h4h = 0x0f6d2b69; + this.#h4l = 0x7bd44da8; + this.#h5h = 0x77e36f73; + this.#h5l = 0x04c48942; + this.#h6h = 0x3f9d85a8; + this.#h6l = 0x6a1d36c8; + this.#h7h = 0x1112e6ad; + this.#h7l = 0x91d692a1; + } else if (bits === 256) { + this.#h0h = 0x22312194; + this.#h0l = 0xfc2bf72c; + this.#h1h = 0x9f555fa3; + this.#h1l = 0xc84c64c2; + this.#h2h = 0x2393b86b; + this.#h2l = 0x6f53b151; + this.#h3h = 0x96387719; + this.#h3l = 0x5940eabd; + this.#h4h = 0x96283ee2; + this.#h4l = 0xa88effe3; + this.#h5h = 0xbe5e1e25; + this.#h5l = 0x53863992; + this.#h6h = 0x2b0199fc; + this.#h6l = 0x2c85b8aa; + this.#h7h = 0x0eb72ddc; + this.#h7l = 0x81c52ca2; + } else if (bits === 384) { + this.#h0h = 0xcbbb9d5d; + this.#h0l = 0xc1059ed8; + this.#h1h = 0x629a292a; + this.#h1l = 0x367cd507; + this.#h2h = 0x9159015a; + this.#h2l = 0x3070dd17; + this.#h3h = 0x152fecd8; + this.#h3l = 0xf70e5939; + this.#h4h = 0x67332667; + this.#h4l = 0xffc00b31; + this.#h5h = 0x8eb44a87; + this.#h5l = 0x68581511; + this.#h6h = 0xdb0c2e0d; + this.#h6l = 0x64f98fa7; + this.#h7h = 0x47b5481d; + this.#h7l = 0xbefa4fa4; + } else { // 512 + this.#h0h = 0x6a09e667; + this.#h0l = 0xf3bcc908; + this.#h1h = 0xbb67ae85; + this.#h1l = 0x84caa73b; + this.#h2h = 0x3c6ef372; + this.#h2l = 0xfe94f82b; + this.#h3h = 0xa54ff53a; + this.#h3l = 0x5f1d36f1; + this.#h4h = 0x510e527f; + this.#h4l = 0xade682d1; + this.#h5h = 0x9b05688c; + this.#h5l = 0x2b3e6c1f; + this.#h6h = 0x1f83d9ab; + this.#h6l = 0xfb41bd6b; + this.#h7h = 0x5be0cd19; + this.#h7l = 0x137e2179; + } + this.#bits = bits; + this.#block = this.#start = this.#bytes = this.#hBytes = 0; + this.#finalized = this.#hashed = false; + } + + update(message: Message): this { + if (this.#finalized) { + return this; + } + let msg: string | number[] | Uint8Array; + if (message instanceof ArrayBuffer) { + msg = new Uint8Array(message); + } else { + msg = message; + } + const length = msg.length; + const blocks = this.#blocks; + let index = 0; + while (index < length) { + let i: number; + if (this.#hashed) { + this.#hashed = false; + blocks[0] = this.#block; + blocks[1] = blocks[2] = blocks[3] = blocks[4] = blocks[5] = blocks[6] = blocks[7] = blocks[8] = + blocks[9] = blocks[10] = blocks[11] = blocks[12] = blocks[13] = blocks[14] = blocks[15] = blocks[16] = + blocks[17] = blocks[18] = blocks[19] = blocks[20] = blocks[21] = blocks[22] = blocks[23] = blocks[24] = + blocks[25] = blocks[26] = blocks[27] = blocks[28] = blocks[29] = blocks[30] = blocks[31] = blocks[32] = 0; + } + if (typeof msg !== "string") { + for (i = this.#start; index < length && i < 128; ++index) { + blocks[i >> 2] |= msg[index] << SHIFT[i++ & 3]; + } + } else { + for (i = this.#start; index < length && i < 128; ++index) { + let code = msg.charCodeAt(index); + if (code < 0x80) { + blocks[i >> 2] |= code << SHIFT[i++ & 3]; + } else if (code < 0x800) { + blocks[i >> 2] |= (0xc0 | (code >> 6)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | (code & 0x3f)) << SHIFT[i++ & 3]; + } else if (code < 0xd800 || code >= 0xe000) { + blocks[i >> 2] |= (0xe0 | (code >> 12)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | ((code >> 6) & 0x3f)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | (code & 0x3f)) << SHIFT[i++ & 3]; + } else { + code = 0x10000 + (((code & 0x3ff) << 10) | (msg.charCodeAt(++index) & 0x3ff)); + blocks[i >> 2] |= (0xf0 | (code >> 18)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | ((code >> 12) & 0x3f)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | ((code >> 6) & 0x3f)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | (code & 0x3f)) << SHIFT[i++ & 3]; + } + } + } + this.#lastByteIndex = i; + this.#bytes += i - this.#start; + if (i >= 128) { + this.#block = blocks[32]; + this.#start = i - 128; + this.hash(); + this.#hashed = true; + } else { + this.#start = i; + } + } + if (this.#bytes > 4294967295) { + this.#hBytes += (this.#bytes / 4294967296) << 0; + this.#bytes = this.#bytes % 4294967296; + } + return this; + } + + protected finalize(): void { + if (this.#finalized) { + return; + } + this.#finalized = true; + const blocks = this.#blocks; + const i = this.#lastByteIndex; + blocks[32] = this.#block; + blocks[i >> 2] |= EXTRA[i & 3]; + this.#block = blocks[32]; + if (i >= 112) { + if (!this.#hashed) { + this.hash(); + } + blocks[0] = this.#block; + blocks[1] = blocks[2] = blocks[3] = blocks[4] = blocks[5] = blocks[6] = blocks[7] = blocks[8] = + blocks[9] =blocks[10] = blocks[11] = blocks[12] = blocks[13] = blocks[14] = blocks[15] = blocks[16] = + blocks[17] = blocks[18] = blocks[19] = blocks[20] = blocks[21] = blocks[22] = blocks[23] = blocks[24] = + blocks[25] = blocks[26] = blocks[27] = blocks[28] = blocks[29] = blocks[30] = blocks[31] = blocks[32] = 0; + } + blocks[30] = (this.#hBytes << 3) | (this.#bytes >>> 29); + blocks[31] = this.#bytes << 3; + this.hash(); + } + + protected hash(): void { + const + h0h = this.#h0h, h0l = this.#h0l, h1h = this.#h1h, h1l = this.#h1l, h2h = this.#h2h, h2l = this.#h2l, + h3h = this.#h3h, h3l = this.#h3l, h4h = this.#h4h, h4l = this.#h4l, h5h = this.#h5h, h5l = this.#h5l, + h6h = this.#h6h, h6l = this.#h6l, h7h = this.#h7h, h7l = this.#h7l; + + let s0h, s0l, s1h, s1l, c1, c2, c3, c4, abh, abl, dah, dal, cdh, cdl, bch, bcl, majh, majl, + t1h, t1l, t2h, t2l, chh, chl: number; + + const blocks = this.#blocks; + + for (let j = 32; j < 160; j += 2) { + t1h = blocks[j - 30]; + t1l = blocks[j - 29]; + s0h = ((t1h >>> 1) | (t1l << 31)) ^ ((t1h >>> 8) | (t1l << 24)) ^ (t1h >>> 7); + s0l = ((t1l >>> 1) | (t1h << 31)) ^ ((t1l >>> 8) | (t1h << 24)) ^ ((t1l >>> 7) | (t1h << 25)); + + t1h = blocks[j - 4]; + t1l = blocks[j - 3]; + s1h = ((t1h >>> 19) | (t1l << 13)) ^ ((t1l >>> 29) | (t1h << 3)) ^ (t1h >>> 6); + s1l = ((t1l >>> 19) | (t1h << 13)) ^ ((t1h >>> 29) | (t1l << 3)) ^ ((t1l >>> 6) | (t1h << 26)); + + t1h = blocks[j - 32]; + t1l = blocks[j - 31]; + t2h = blocks[j - 14]; + t2l = blocks[j - 13]; + + c1 = (t2l & 0xffff) + (t1l & 0xffff) + (s0l & 0xffff) + (s1l & 0xffff); + c2 = (t2l >>> 16) + (t1l >>> 16) + (s0l >>> 16) + (s1l >>> 16) + (c1 >>> 16); + c3 = (t2h & 0xffff) + (t1h & 0xffff) + (s0h & 0xffff) + (s1h & 0xffff) + (c2 >>> 16); + c4 = (t2h >>> 16) + (t1h >>> 16) + (s0h >>> 16) + (s1h >>> 16) + (c3 >>> 16); + + blocks[j] = (c4 << 16) | (c3 & 0xffff); + blocks[j + 1] = (c2 << 16) | (c1 & 0xffff); + } + + let ah = h0h, al = h0l, bh = h1h, bl = h1l, ch = h2h, cl = h2l, dh = h3h, dl = h3l, eh = h4h, el = h4l, + fh = h5h, fl = h5l, gh = h6h, gl = h6l, hh = h7h, hl = h7l; + + bch = bh & ch; + bcl = bl & cl; + + for (let j = 0; j < 160; j += 8) { + s0h = ((ah >>> 28) | (al << 4)) ^ ((al >>> 2) | (ah << 30)) ^ ((al >>> 7) | (ah << 25)); + s0l = ((al >>> 28) | (ah << 4)) ^ ((ah >>> 2) | (al << 30)) ^ ((ah >>> 7) | (al << 25)); + + s1h = ((eh >>> 14) | (el << 18)) ^ ((eh >>> 18) | (el << 14)) ^ ((el >>> 9) | (eh << 23)); + s1l = ((el >>> 14) | (eh << 18)) ^ ((el >>> 18) | (eh << 14)) ^ ((eh >>> 9) | (el << 23)); + + abh = ah & bh; + abl = al & bl; + majh = abh ^ (ah & ch) ^ bch; + majl = abl ^ (al & cl) ^ bcl; + + chh = (eh & fh) ^ (~eh & gh); + chl = (el & fl) ^ (~el & gl); + + t1h = blocks[j]; + t1l = blocks[j + 1]; + t2h = K[j]; + t2l = K[j + 1]; + + c1 = (t2l & 0xffff) + (t1l & 0xffff) + (chl & 0xffff) + (s1l & 0xffff) + (hl & 0xffff); + c2 = (t2l >>> 16) + (t1l >>> 16) + (chl >>> 16) + (s1l >>> 16) + (hl >>> 16) + (c1 >>> 16); + c3 = (t2h & 0xffff) + (t1h & 0xffff) + (chh & 0xffff) + (s1h & 0xffff) + (hh & 0xffff) + (c2 >>> 16); + c4 = (t2h >>> 16) + (t1h >>> 16) + (chh >>> 16) + (s1h >>> 16) + (hh >>> 16) + (c3 >>> 16); + + t1h = (c4 << 16) | (c3 & 0xffff); + t1l = (c2 << 16) | (c1 & 0xffff); + + c1 = (majl & 0xffff) + (s0l & 0xffff); + c2 = (majl >>> 16) + (s0l >>> 16) + (c1 >>> 16); + c3 = (majh & 0xffff) + (s0h & 0xffff) + (c2 >>> 16); + c4 = (majh >>> 16) + (s0h >>> 16) + (c3 >>> 16); + + t2h = (c4 << 16) | (c3 & 0xffff); + t2l = (c2 << 16) | (c1 & 0xffff); + + c1 = (dl & 0xffff) + (t1l & 0xffff); + c2 = (dl >>> 16) + (t1l >>> 16) + (c1 >>> 16); + c3 = (dh & 0xffff) + (t1h & 0xffff) + (c2 >>> 16); + c4 = (dh >>> 16) + (t1h >>> 16) + (c3 >>> 16); + + hh = (c4 << 16) | (c3 & 0xffff); + hl = (c2 << 16) | (c1 & 0xffff); + + c1 = (t2l & 0xffff) + (t1l & 0xffff); + c2 = (t2l >>> 16) + (t1l >>> 16) + (c1 >>> 16); + c3 = (t2h & 0xffff) + (t1h & 0xffff) + (c2 >>> 16); + c4 = (t2h >>> 16) + (t1h >>> 16) + (c3 >>> 16); + + dh = (c4 << 16) | (c3 & 0xffff); + dl = (c2 << 16) | (c1 & 0xffff); + + s0h = ((dh >>> 28) | (dl << 4)) ^ ((dl >>> 2) | (dh << 30)) ^ ((dl >>> 7) | (dh << 25)); + s0l = ((dl >>> 28) | (dh << 4)) ^ ((dh >>> 2) | (dl << 30)) ^ ((dh >>> 7) | (dl << 25)); + + s1h = ((hh >>> 14) | (hl << 18)) ^ ((hh >>> 18) | (hl << 14)) ^ ((hl >>> 9) | (hh << 23)); + s1l = ((hl >>> 14) | (hh << 18)) ^ ((hl >>> 18) | (hh << 14)) ^ ((hh >>> 9) | (hl << 23)); + + dah = dh & ah; + dal = dl & al; + majh = dah ^ (dh & bh) ^ abh; + majl = dal ^ (dl & bl) ^ abl; + + chh = (hh & eh) ^ (~hh & fh); + chl = (hl & el) ^ (~hl & fl); + + t1h = blocks[j + 2]; + t1l = blocks[j + 3]; + t2h = K[j + 2]; + t2l = K[j + 3]; + + c1 = (t2l & 0xffff) + (t1l & 0xffff) + (chl & 0xffff) + (s1l & 0xffff) + (gl & 0xffff); + c2 = (t2l >>> 16) + (t1l >>> 16) + (chl >>> 16) + (s1l >>> 16) + (gl >>> 16) + (c1 >>> 16); + c3 = (t2h & 0xffff) + (t1h & 0xffff) + (chh & 0xffff) + (s1h & 0xffff) + (gh & 0xffff) + (c2 >>> 16); + c4 = (t2h >>> 16) + (t1h >>> 16) + (chh >>> 16) + (s1h >>> 16) + (gh >>> 16) + (c3 >>> 16); + + t1h = (c4 << 16) | (c3 & 0xffff); + t1l = (c2 << 16) | (c1 & 0xffff); + + c1 = (majl & 0xffff) + (s0l & 0xffff); + c2 = (majl >>> 16) + (s0l >>> 16) + (c1 >>> 16); + c3 = (majh & 0xffff) + (s0h & 0xffff) + (c2 >>> 16); + c4 = (majh >>> 16) + (s0h >>> 16) + (c3 >>> 16); + + t2h = (c4 << 16) | (c3 & 0xffff); + t2l = (c2 << 16) | (c1 & 0xffff); + + c1 = (cl & 0xffff) + (t1l & 0xffff); + c2 = (cl >>> 16) + (t1l >>> 16) + (c1 >>> 16); + c3 = (ch & 0xffff) + (t1h & 0xffff) + (c2 >>> 16); + c4 = (ch >>> 16) + (t1h >>> 16) + (c3 >>> 16); + + gh = (c4 << 16) | (c3 & 0xffff); + gl = (c2 << 16) | (c1 & 0xffff); + + c1 = (t2l & 0xffff) + (t1l & 0xffff); + c2 = (t2l >>> 16) + (t1l >>> 16) + (c1 >>> 16); + c3 = (t2h & 0xffff) + (t1h & 0xffff) + (c2 >>> 16); + c4 = (t2h >>> 16) + (t1h >>> 16) + (c3 >>> 16); + + ch = (c4 << 16) | (c3 & 0xffff); + cl = (c2 << 16) | (c1 & 0xffff); + + s0h = ((ch >>> 28) | (cl << 4)) ^ ((cl >>> 2) | (ch << 30)) ^ ((cl >>> 7) | (ch << 25)); + s0l = ((cl >>> 28) | (ch << 4)) ^ ((ch >>> 2) | (cl << 30)) ^ ((ch >>> 7) | (cl << 25)); + + s1h = ((gh >>> 14) | (gl << 18)) ^ ((gh >>> 18) | (gl << 14)) ^ ((gl >>> 9) | (gh << 23)); + s1l = ((gl >>> 14) | (gh << 18)) ^ ((gl >>> 18) | (gh << 14)) ^ ((gh >>> 9) | (gl << 23)); + + cdh = ch & dh; + cdl = cl & dl; + majh = cdh ^ (ch & ah) ^ dah; + majl = cdl ^ (cl & al) ^ dal; + + chh = (gh & hh) ^ (~gh & eh); + chl = (gl & hl) ^ (~gl & el); + + t1h = blocks[j + 4]; + t1l = blocks[j + 5]; + t2h = K[j + 4]; + t2l = K[j + 5]; + + c1 = (t2l & 0xffff) + (t1l & 0xffff) + (chl & 0xffff) + (s1l & 0xffff) + (fl & 0xffff); + c2 = (t2l >>> 16) + (t1l >>> 16) + (chl >>> 16) + (s1l >>> 16) + (fl >>> 16) + (c1 >>> 16); + c3 = (t2h & 0xffff) + (t1h & 0xffff) + (chh & 0xffff) + (s1h & 0xffff) + (fh & 0xffff) + (c2 >>> 16); + c4 = (t2h >>> 16) + (t1h >>> 16) + (chh >>> 16) + (s1h >>> 16) + (fh >>> 16) + (c3 >>> 16); + + t1h = (c4 << 16) | (c3 & 0xffff); + t1l = (c2 << 16) | (c1 & 0xffff); + + c1 = (majl & 0xffff) + (s0l & 0xffff); + c2 = (majl >>> 16) + (s0l >>> 16) + (c1 >>> 16); + c3 = (majh & 0xffff) + (s0h & 0xffff) + (c2 >>> 16); + c4 = (majh >>> 16) + (s0h >>> 16) + (c3 >>> 16); + + t2h = (c4 << 16) | (c3 & 0xffff); + t2l = (c2 << 16) | (c1 & 0xffff); + + c1 = (bl & 0xffff) + (t1l & 0xffff); + c2 = (bl >>> 16) + (t1l >>> 16) + (c1 >>> 16); + c3 = (bh & 0xffff) + (t1h & 0xffff) + (c2 >>> 16); + c4 = (bh >>> 16) + (t1h >>> 16) + (c3 >>> 16); + + fh = (c4 << 16) | (c3 & 0xffff); + fl = (c2 << 16) | (c1 & 0xffff); + + c1 = (t2l & 0xffff) + (t1l & 0xffff); + c2 = (t2l >>> 16) + (t1l >>> 16) + (c1 >>> 16); + c3 = (t2h & 0xffff) + (t1h & 0xffff) + (c2 >>> 16); + c4 = (t2h >>> 16) + (t1h >>> 16) + (c3 >>> 16); + + bh = (c4 << 16) | (c3 & 0xffff); + bl = (c2 << 16) | (c1 & 0xffff); + + s0h = ((bh >>> 28) | (bl << 4)) ^ ((bl >>> 2) | (bh << 30)) ^ ((bl >>> 7) | (bh << 25)); + s0l = ((bl >>> 28) | (bh << 4)) ^ ((bh >>> 2) | (bl << 30)) ^ ((bh >>> 7) | (bl << 25)); + + s1h = ((fh >>> 14) | (fl << 18)) ^ ((fh >>> 18) | (fl << 14)) ^ ((fl >>> 9) | (fh << 23)); + s1l = ((fl >>> 14) | (fh << 18)) ^ ((fl >>> 18) | (fh << 14)) ^ ((fh >>> 9) | (fl << 23)); + + bch = bh & ch; + bcl = bl & cl; + majh = bch ^ (bh & dh) ^ cdh; + majl = bcl ^ (bl & dl) ^ cdl; + + chh = (fh & gh) ^ (~fh & hh); + chl = (fl & gl) ^ (~fl & hl); + + t1h = blocks[j + 6]; + t1l = blocks[j + 7]; + t2h = K[j + 6]; + t2l = K[j + 7]; + + c1 = (t2l & 0xffff) + (t1l & 0xffff) + (chl & 0xffff) + (s1l & 0xffff) + (el & 0xffff); + c2 = (t2l >>> 16) + (t1l >>> 16) + (chl >>> 16) + (s1l >>> 16) + (el >>> 16) + (c1 >>> 16); + c3 = (t2h & 0xffff) + (t1h & 0xffff) + (chh & 0xffff) + (s1h & 0xffff) + (eh & 0xffff) + (c2 >>> 16); + c4 = (t2h >>> 16) + (t1h >>> 16) + (chh >>> 16) + (s1h >>> 16) + (eh >>> 16) + (c3 >>> 16); + + t1h = (c4 << 16) | (c3 & 0xffff); + t1l = (c2 << 16) | (c1 & 0xffff); + + c1 = (majl & 0xffff) + (s0l & 0xffff); + c2 = (majl >>> 16) + (s0l >>> 16) + (c1 >>> 16); + c3 = (majh & 0xffff) + (s0h & 0xffff) + (c2 >>> 16); + c4 = (majh >>> 16) + (s0h >>> 16) + (c3 >>> 16); + + t2h = (c4 << 16) | (c3 & 0xffff); + t2l = (c2 << 16) | (c1 & 0xffff); + + c1 = (al & 0xffff) + (t1l & 0xffff); + c2 = (al >>> 16) + (t1l >>> 16) + (c1 >>> 16); + c3 = (ah & 0xffff) + (t1h & 0xffff) + (c2 >>> 16); + c4 = (ah >>> 16) + (t1h >>> 16) + (c3 >>> 16); + + eh = (c4 << 16) | (c3 & 0xffff); + el = (c2 << 16) | (c1 & 0xffff); + + c1 = (t2l & 0xffff) + (t1l & 0xffff); + c2 = (t2l >>> 16) + (t1l >>> 16) + (c1 >>> 16); + c3 = (t2h & 0xffff) + (t1h & 0xffff) + (c2 >>> 16); + c4 = (t2h >>> 16) + (t1h >>> 16) + (c3 >>> 16); + + ah = (c4 << 16) | (c3 & 0xffff); + al = (c2 << 16) | (c1 & 0xffff); + } + + c1 = (h0l & 0xffff) + (al & 0xffff); + c2 = (h0l >>> 16) + (al >>> 16) + (c1 >>> 16); + c3 = (h0h & 0xffff) + (ah & 0xffff) + (c2 >>> 16); + c4 = (h0h >>> 16) + (ah >>> 16) + (c3 >>> 16); + + this.#h0h = (c4 << 16) | (c3 & 0xffff); + this.#h0l = (c2 << 16) | (c1 & 0xffff); + + c1 = (h1l & 0xffff) + (bl & 0xffff); + c2 = (h1l >>> 16) + (bl >>> 16) + (c1 >>> 16); + c3 = (h1h & 0xffff) + (bh & 0xffff) + (c2 >>> 16); + c4 = (h1h >>> 16) + (bh >>> 16) + (c3 >>> 16); + + this.#h1h = (c4 << 16) | (c3 & 0xffff); + this.#h1l = (c2 << 16) | (c1 & 0xffff); + + c1 = (h2l & 0xffff) + (cl & 0xffff); + c2 = (h2l >>> 16) + (cl >>> 16) + (c1 >>> 16); + c3 = (h2h & 0xffff) + (ch & 0xffff) + (c2 >>> 16); + c4 = (h2h >>> 16) + (ch >>> 16) + (c3 >>> 16); + + this.#h2h = (c4 << 16) | (c3 & 0xffff); + this.#h2l = (c2 << 16) | (c1 & 0xffff); + + c1 = (h3l & 0xffff) + (dl & 0xffff); + c2 = (h3l >>> 16) + (dl >>> 16) + (c1 >>> 16); + c3 = (h3h & 0xffff) + (dh & 0xffff) + (c2 >>> 16); + c4 = (h3h >>> 16) + (dh >>> 16) + (c3 >>> 16); + + this.#h3h = (c4 << 16) | (c3 & 0xffff); + this.#h3l = (c2 << 16) | (c1 & 0xffff); + + c1 = (h4l & 0xffff) + (el & 0xffff); + c2 = (h4l >>> 16) + (el >>> 16) + (c1 >>> 16); + c3 = (h4h & 0xffff) + (eh & 0xffff) + (c2 >>> 16); + c4 = (h4h >>> 16) + (eh >>> 16) + (c3 >>> 16); + + this.#h4h = (c4 << 16) | (c3 & 0xffff); + this.#h4l = (c2 << 16) | (c1 & 0xffff); + + c1 = (h5l & 0xffff) + (fl & 0xffff); + c2 = (h5l >>> 16) + (fl >>> 16) + (c1 >>> 16); + c3 = (h5h & 0xffff) + (fh & 0xffff) + (c2 >>> 16); + c4 = (h5h >>> 16) + (fh >>> 16) + (c3 >>> 16); + + this.#h5h = (c4 << 16) | (c3 & 0xffff); + this.#h5l = (c2 << 16) | (c1 & 0xffff); + + c1 = (h6l & 0xffff) + (gl & 0xffff); + c2 = (h6l >>> 16) + (gl >>> 16) + (c1 >>> 16); + c3 = (h6h & 0xffff) + (gh & 0xffff) + (c2 >>> 16); + c4 = (h6h >>> 16) + (gh >>> 16) + (c3 >>> 16); + + this.#h6h = (c4 << 16) | (c3 & 0xffff); + this.#h6l = (c2 << 16) | (c1 & 0xffff); + + c1 = (h7l & 0xffff) + (hl & 0xffff); + c2 = (h7l >>> 16) + (hl >>> 16) + (c1 >>> 16); + c3 = (h7h & 0xffff) + (hh & 0xffff) + (c2 >>> 16); + c4 = (h7h >>> 16) + (hh >>> 16) + (c3 >>> 16); + + this.#h7h = (c4 << 16) | (c3 & 0xffff); + this.#h7l = (c2 << 16) | (c1 & 0xffff); + } + + hex(): string { + this.finalize(); + const + h0h = this.#h0h, h0l = this.#h0l, h1h = this.#h1h, h1l = this.#h1l, h2h = this.#h2h, h2l = this.#h2l, + h3h = this.#h3h, h3l = this.#h3l, h4h = this.#h4h, h4l = this.#h4l, h5h = this.#h5h, h5l = this.#h5l, + h6h = this.#h6h, h6l = this.#h6l, h7h = this.#h7h, h7l = this.#h7l, bits = this.#bits; + let hex = + HEX_CHARS[(h0h >> 28) & 0x0f] + HEX_CHARS[(h0h >> 24) & 0x0f] + + HEX_CHARS[(h0h >> 20) & 0x0f] + HEX_CHARS[(h0h >> 16) & 0x0f] + + HEX_CHARS[(h0h >> 12) & 0x0f] + HEX_CHARS[(h0h >> 8) & 0x0f] + + HEX_CHARS[(h0h >> 4) & 0x0f] + HEX_CHARS[h0h & 0x0f] + + HEX_CHARS[(h0l >> 28) & 0x0f] + HEX_CHARS[(h0l >> 24) & 0x0f] + + HEX_CHARS[(h0l >> 20) & 0x0f] + HEX_CHARS[(h0l >> 16) & 0x0f] + + HEX_CHARS[(h0l >> 12) & 0x0f] + HEX_CHARS[(h0l >> 8) & 0x0f] + + HEX_CHARS[(h0l >> 4) & 0x0f] + HEX_CHARS[h0l & 0x0f] + + HEX_CHARS[(h1h >> 28) & 0x0f] + HEX_CHARS[(h1h >> 24) & 0x0f] + + HEX_CHARS[(h1h >> 20) & 0x0f] + HEX_CHARS[(h1h >> 16) & 0x0f] + + HEX_CHARS[(h1h >> 12) & 0x0f] + HEX_CHARS[(h1h >> 8) & 0x0f] + + HEX_CHARS[(h1h >> 4) & 0x0f] + HEX_CHARS[h1h & 0x0f] + + HEX_CHARS[(h1l >> 28) & 0x0f] + HEX_CHARS[(h1l >> 24) & 0x0f] + + HEX_CHARS[(h1l >> 20) & 0x0f] + HEX_CHARS[(h1l >> 16) & 0x0f] + + HEX_CHARS[(h1l >> 12) & 0x0f] + HEX_CHARS[(h1l >> 8) & 0x0f] + + HEX_CHARS[(h1l >> 4) & 0x0f] + HEX_CHARS[h1l & 0x0f] + + HEX_CHARS[(h2h >> 28) & 0x0f] + HEX_CHARS[(h2h >> 24) & 0x0f] + + HEX_CHARS[(h2h >> 20) & 0x0f] + HEX_CHARS[(h2h >> 16) & 0x0f] + + HEX_CHARS[(h2h >> 12) & 0x0f] + HEX_CHARS[(h2h >> 8) & 0x0f] + + HEX_CHARS[(h2h >> 4) & 0x0f] + HEX_CHARS[h2h & 0x0f] + + HEX_CHARS[(h2l >> 28) & 0x0f] + HEX_CHARS[(h2l >> 24) & 0x0f] + + HEX_CHARS[(h2l >> 20) & 0x0f] + HEX_CHARS[(h2l >> 16) & 0x0f] + + HEX_CHARS[(h2l >> 12) & 0x0f] + HEX_CHARS[(h2l >> 8) & 0x0f] + + HEX_CHARS[(h2l >> 4) & 0x0f] + HEX_CHARS[h2l & 0x0f] + + HEX_CHARS[(h3h >> 28) & 0x0f] + HEX_CHARS[(h3h >> 24) & 0x0f] + + HEX_CHARS[(h3h >> 20) & 0x0f] + HEX_CHARS[(h3h >> 16) & 0x0f] + + HEX_CHARS[(h3h >> 12) & 0x0f] + HEX_CHARS[(h3h >> 8) & 0x0f] + + HEX_CHARS[(h3h >> 4) & 0x0f] + HEX_CHARS[h3h & 0x0f]; + if (bits >= 256) { + hex += + HEX_CHARS[(h3l >> 28) & 0x0f] + HEX_CHARS[(h3l >> 24) & 0x0f] + + HEX_CHARS[(h3l >> 20) & 0x0f] + HEX_CHARS[(h3l >> 16) & 0x0f] + + HEX_CHARS[(h3l >> 12) & 0x0f] + HEX_CHARS[(h3l >> 8) & 0x0f] + + HEX_CHARS[(h3l >> 4) & 0x0f] + HEX_CHARS[h3l & 0x0f]; + } + if (bits >= 384) { + hex += + HEX_CHARS[(h4h >> 28) & 0x0f] + HEX_CHARS[(h4h >> 24) & 0x0f] + + HEX_CHARS[(h4h >> 20) & 0x0f] + HEX_CHARS[(h4h >> 16) & 0x0f] + + HEX_CHARS[(h4h >> 12) & 0x0f] + HEX_CHARS[(h4h >> 8) & 0x0f] + + HEX_CHARS[(h4h >> 4) & 0x0f] + HEX_CHARS[h4h & 0x0f] + + HEX_CHARS[(h4l >> 28) & 0x0f] + HEX_CHARS[(h4l >> 24) & 0x0f] + + HEX_CHARS[(h4l >> 20) & 0x0f] + HEX_CHARS[(h4l >> 16) & 0x0f] + + HEX_CHARS[(h4l >> 12) & 0x0f] + HEX_CHARS[(h4l >> 8) & 0x0f] + + HEX_CHARS[(h4l >> 4) & 0x0f] + HEX_CHARS[h4l & 0x0f] + + HEX_CHARS[(h5h >> 28) & 0x0f] + HEX_CHARS[(h5h >> 24) & 0x0f] + + HEX_CHARS[(h5h >> 20) & 0x0f] + HEX_CHARS[(h5h >> 16) & 0x0f] + + HEX_CHARS[(h5h >> 12) & 0x0f] + HEX_CHARS[(h5h >> 8) & 0x0f] + + HEX_CHARS[(h5h >> 4) & 0x0f] + HEX_CHARS[h5h & 0x0f] + + HEX_CHARS[(h5l >> 28) & 0x0f] + HEX_CHARS[(h5l >> 24) & 0x0f] + + HEX_CHARS[(h5l >> 20) & 0x0f] + HEX_CHARS[(h5l >> 16) & 0x0f] + + HEX_CHARS[(h5l >> 12) & 0x0f] + HEX_CHARS[(h5l >> 8) & 0x0f] + + HEX_CHARS[(h5l >> 4) & 0x0f] + HEX_CHARS[h5l & 0x0f]; + } + if (bits === 512) { + hex += + HEX_CHARS[(h6h >> 28) & 0x0f] + HEX_CHARS[(h6h >> 24) & 0x0f] + + HEX_CHARS[(h6h >> 20) & 0x0f] + HEX_CHARS[(h6h >> 16) & 0x0f] + + HEX_CHARS[(h6h >> 12) & 0x0f] + HEX_CHARS[(h6h >> 8) & 0x0f] + + HEX_CHARS[(h6h >> 4) & 0x0f] + HEX_CHARS[h6h & 0x0f] + + HEX_CHARS[(h6l >> 28) & 0x0f] + HEX_CHARS[(h6l >> 24) & 0x0f] + + HEX_CHARS[(h6l >> 20) & 0x0f] + HEX_CHARS[(h6l >> 16) & 0x0f] + + HEX_CHARS[(h6l >> 12) & 0x0f] + HEX_CHARS[(h6l >> 8) & 0x0f] + + HEX_CHARS[(h6l >> 4) & 0x0f] + HEX_CHARS[h6l & 0x0f] + + HEX_CHARS[(h7h >> 28) & 0x0f] + HEX_CHARS[(h7h >> 24) & 0x0f] + + HEX_CHARS[(h7h >> 20) & 0x0f] + HEX_CHARS[(h7h >> 16) & 0x0f] + + HEX_CHARS[(h7h >> 12) & 0x0f] + HEX_CHARS[(h7h >> 8) & 0x0f] + + HEX_CHARS[(h7h >> 4) & 0x0f] + HEX_CHARS[h7h & 0x0f] + + HEX_CHARS[(h7l >> 28) & 0x0f] + HEX_CHARS[(h7l >> 24) & 0x0f] + + HEX_CHARS[(h7l >> 20) & 0x0f] + HEX_CHARS[(h7l >> 16) & 0x0f] + + HEX_CHARS[(h7l >> 12) & 0x0f] + HEX_CHARS[(h7l >> 8) & 0x0f] + + HEX_CHARS[(h7l >> 4) & 0x0f] + HEX_CHARS[h7l & 0x0f]; + } + return hex; + } + + toString(): string { + return this.hex(); + } + + digest(): number[] { + this.finalize(); + const + h0h = this.#h0h, h0l = this.#h0l, h1h = this.#h1h, h1l = this.#h1l, h2h = this.#h2h, h2l = this.#h2l, + h3h = this.#h3h, h3l = this.#h3l, h4h = this.#h4h, h4l = this.#h4l, h5h = this.#h5h, h5l = this.#h5l, + h6h = this.#h6h, h6l = this.#h6l, h7h = this.#h7h, h7l = this.#h7l, bits = this.#bits; + const arr = [ + (h0h >> 24) & 0xff, (h0h >> 16) & 0xff, (h0h >> 8) & 0xff, h0h & 0xff, + (h0l >> 24) & 0xff, (h0l >> 16) & 0xff, (h0l >> 8) & 0xff, h0l & 0xff, + (h1h >> 24) & 0xff, (h1h >> 16) & 0xff, (h1h >> 8) & 0xff, h1h & 0xff, + (h1l >> 24) & 0xff, (h1l >> 16) & 0xff, (h1l >> 8) & 0xff, h1l & 0xff, + (h2h >> 24) & 0xff, (h2h >> 16) & 0xff, (h2h >> 8) & 0xff, h2h & 0xff, + (h2l >> 24) & 0xff, (h2l >> 16) & 0xff, (h2l >> 8) & 0xff, h2l & 0xff, + (h3h >> 24) & 0xff, (h3h >> 16) & 0xff, (h3h >> 8) & 0xff, h3h & 0xff + ]; + if (bits >= 256) { + arr.push((h3l >> 24) & 0xff, (h3l >> 16) & 0xff, (h3l >> 8) & 0xff, h3l & 0xff); + } + if (bits >= 384) { + arr.push( + (h4h >> 24) & 0xff, (h4h >> 16) & 0xff, (h4h >> 8) & 0xff, h4h & 0xff, + (h4l >> 24) & 0xff, (h4l >> 16) & 0xff, (h4l >> 8) & 0xff, h4l & 0xff, + (h5h >> 24) & 0xff, (h5h >> 16) & 0xff, (h5h >> 8) & 0xff, h5h & 0xff, + (h5l >> 24) & 0xff, (h5l >> 16) & 0xff, (h5l >> 8) & 0xff, h5l & 0xff + ); + } + if (bits === 512) { + arr.push( + (h6h >> 24) & 0xff, (h6h >> 16) & 0xff, (h6h >> 8) & 0xff, h6h & 0xff, + (h6l >> 24) & 0xff, (h6l >> 16) & 0xff, (h6l >> 8) & 0xff, h6l & 0xff, + (h7h >> 24) & 0xff, (h7h >> 16) & 0xff, (h7h >> 8) & 0xff, h7h & 0xff, + (h7l >> 24) & 0xff, (h7l >> 16) & 0xff, (h7l >> 8) & 0xff, h7l & 0xff + ); + } + return arr; + } + + array(): number[] { + return this.digest(); + } + + arrayBuffer(): ArrayBuffer { + this.finalize(); + const bits = this.#bits; + const buffer = new ArrayBuffer(bits / 8); + const dataView = new DataView(buffer); + dataView.setUint32(0, this.#h0h); + dataView.setUint32(4, this.#h0l); + dataView.setUint32(8, this.#h1h); + dataView.setUint32(12, this.#h1l); + dataView.setUint32(16, this.#h2h); + dataView.setUint32(20, this.#h2l); + dataView.setUint32(24, this.#h3h); + if (bits >= 256) { + dataView.setUint32(28, this.#h3l); + } + if (bits >= 384) { + dataView.setUint32(32, this.#h4h); + dataView.setUint32(36, this.#h4l); + dataView.setUint32(40, this.#h5h); + dataView.setUint32(44, this.#h5l); + } + if (bits === 512) { + dataView.setUint32(48, this.#h6h); + dataView.setUint32(52, this.#h6l); + dataView.setUint32(56, this.#h7h); + dataView.setUint32(60, this.#h7l); + } + return buffer; + } +} + +export class HmacSha512 extends Sha512 { + #inner: boolean; + #bits: number; + #oKeyPad: number[]; + #sharedMemory: boolean; + + constructor(secretKey: Message, bits = 512, sharedMemory = false) { + super(bits, sharedMemory); + + let key: number[] | Uint8Array; + + if (secretKey instanceof ArrayBuffer) { + key = new Uint8Array(secretKey); + } else if (typeof secretKey === "string") { + const bytes: number[] = []; + const length = secretKey.length; + let index = 0; + let code: number; + for (let i = 0; i < length; ++i) { + code = secretKey.charCodeAt(i); + if (code < 0x80) { + bytes[index++] = code; + } else if (code < 0x800) { + bytes[index++] = 0xc0 | (code >> 6); + bytes[index++] = 0x80 | (code & 0x3f); + } else if (code < 0xd800 || code >= 0xe000) { + bytes[index++] = 0xe0 | (code >> 12); + bytes[index++] = 0x80 | ((code >> 6) & 0x3f); + bytes[index++] = 0x80 | (code & 0x3f); + } else { + code = + 0x10000 + + (((code & 0x3ff) << 10) | (secretKey.charCodeAt(++i) & 0x3ff)); + bytes[index++] = 0xf0 | (code >> 18); + bytes[index++] = 0x80 | ((code >> 12) & 0x3f); + bytes[index++] = 0x80 | ((code >> 6) & 0x3f); + bytes[index++] = 0x80 | (code & 0x3f); + } + } + key = bytes; + } else { + key = secretKey; + } + if (key.length > 128) { + key = new Sha512(bits, true).update(key).array(); + } + const oKeyPad: number[] = []; + const iKeyPad: number[] = []; + for (let i = 0; i < 128; ++i) { + const b = key[i] || 0; + oKeyPad[i] = 0x5c ^ b; + iKeyPad[i] = 0x36 ^ b; + } + this.update(iKeyPad); + this.#inner = true; + this.#bits = bits; + this.#oKeyPad = oKeyPad; + this.#sharedMemory = sharedMemory; + } + + protected finalize(): void { + super.finalize(); + if (this.#inner) { + this.#inner = false; + const innerHash = this.array(); + super.init(this.#bits, this.#sharedMemory); + this.update(this.#oKeyPad); + this.update(innerHash); + super.finalize(); + } + } +} diff --git a/std/hash/sha512_test.ts b/std/hash/sha512_test.ts new file mode 100644 index 00000000000000..d17df32307c8ff --- /dev/null +++ b/std/hash/sha512_test.ts @@ -0,0 +1,408 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +import { Sha512, HmacSha512, Message } from "./sha512.ts"; +import { assertEquals } from "../testing/asserts.ts"; +import { join, resolve } from "../path/mod.ts"; + +const { test } = Deno; + +const testdataDir = resolve("hash", "testdata"); + +/** Handy function to convert an array/array buffer to a string of hex values. */ +function toHexString(value: number[] | ArrayBuffer): string { + const array = new Uint8Array(value); + let hex = ""; + for (const v of array) { + const c = v.toString(16); + hex += c.length === 1 ? `0${c}` : c; + } + return hex; +} + +// prettier-ignore +// deno-fmt-ignore +const fixtures: { + sha512bits224: Record>, + sha512bits256: Record>, + sha512: Record>, + hmacSha512bits224: Record>, + hmacSha512bits256: Record>, + hmacSha512: Record> +} = { + sha512bits224: { + "ascii": { + "6ed0dd02806fa89e25de060c19d3ac86cabb87d6a0ddd05c333b84f4": "", + "944cd2847fb54558d4775db0485a50003111c8e5daa63fe722c6aa37": "The quick brown fox jumps over the lazy dog", + "6d6a9279495ec4061769752e7ff9c68b6b0b3c5a281b7917ce0572de": "The quick brown fox jumps over the lazy dog." + }, + "ascii more than 64 bytes": { + "2e962464977b198ee758d615bbc92251ad2e3c0960068e279fd21d2f": "The MD5 message-digest algorithm is a widely used cryptographic hash function producing a 128-bit (16-byte) hash value, typically expressed in text format as a 32 digit hexadecimal number. MD5 has been utilized in a wide variety of cryptographic applications, and is also commonly used to verify data integrity." + }, + "UTF8": { + "0f46a0ae7f226517dd66ece0ce1efa29ffb7ced05ac4566fdcaed188": "中文", + "562f2e4ee7f7451d20dcc6a0ac1a1e1c4a75f09baaf1cf19af3e15f4": "aécio", + "0533318c52b3d4ad355c2a6c7e727ae3d2efa749db480ac33560b059": "𠜎" + }, + "UTF8 more than 64 bytes": { + "f67e191a5d4ee67a272ccaf6cf597f0c4d6a0c46bd631be7cadb0944": "訊息摘要演算法第五版（英語：Message-Digest Algorithm 5，縮寫為MD5），是當前電腦領域用於確保資訊傳輸完整一致而廣泛使用的雜湊演算法之一", + "009c3d1e3172d6df71344982eada855421592aea28acbf660ada7569": "訊息摘要演算法第五版（英語：Message-Digest Algorithm 5，縮寫為MD5），是當前電腦領域用於確保資訊傳輸完整一致而廣泛使用的雜湊演算法之一（又譯雜湊演算法、摘要演算法等），主流程式語言普遍已有MD5的實作。" + }, + "special length": { + "6fe6ce0f03b9cd09851e05ba5e3103df56d2a3dbb379fee437e1cdd3": "0123456780123456780123456780123456780123456780123456780", + "9e6994d879f14c242dea25ebc4d03ae6fc710f5eb60c3962b9dba797": "01234567801234567801234567801234567801234567801234567801", + "204ce3b2af187fe90494cb3e4517257c44917bb7ea6578264baa4fcf": "0123456780123456780123456780123456780123456780123456780123456780", + "69ce912fd1f87e02601d6153c02769ebd7c42b29dcb7963a1c3996da": "01234567801234567801234567801234567801234567801234567801234567801234567", + "bd98be1f148dddd8a98c6ba31628c354456b9754166738fe1aba1037": "012345678012345678012345678012345678012345678012345678012345678012345678" + }, + "Array": { + "6ed0dd02806fa89e25de060c19d3ac86cabb87d6a0ddd05c333b84f4": [], + "6945cf025ed66055282665c546781e32c5a479b5e9b479e96b0c23fe": [211, 212], + "944cd2847fb54558d4775db0485a50003111c8e5daa63fe722c6aa37": [84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103], + "69ce912fd1f87e02601d6153c02769ebd7c42b29dcb7963a1c3996da": [48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55] + }, + "Uint8Array": { + "6945cf025ed66055282665c546781e32c5a479b5e9b479e96b0c23fe": new Uint8Array([211, 212]), + "944cd2847fb54558d4775db0485a50003111c8e5daa63fe722c6aa37": new Uint8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) + }, + "Int8Array": { + "944cd2847fb54558d4775db0485a50003111c8e5daa63fe722c6aa37": new Int8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) + }, + "ArrayBuffer": { + "6ed0dd02806fa89e25de060c19d3ac86cabb87d6a0ddd05c333b84f4": new ArrayBuffer(0), + "283bb59af7081ed08197227d8f65b9591ffe1155be43e9550e57f941": new ArrayBuffer(1) + } + }, + sha512bits256: { + "ascii": { + "c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a": "", + "dd9d67b371519c339ed8dbd25af90e976a1eeefd4ad3d889005e532fc5bef04d": "The quick brown fox jumps over the lazy dog", + "1546741840f8a492b959d9b8b2344b9b0eb51b004bba35c0aebaac86d45264c3": "The quick brown fox jumps over the lazy dog." + }, + "ascii more than 64 bytes": { + "21e2e940930b23f1de6377086d07e22033c6bbf3fd9fbf4b62ec66e6c08c25be": "The MD5 message-digest algorithm is a widely used cryptographic hash function producing a 128-bit (16-byte) hash value, typically expressed in text format as a 32 digit hexadecimal number. MD5 has been utilized in a wide variety of cryptographic applications, and is also commonly used to verify data integrity." + }, + "UTF8": { + "b6dab29c16ec35ab34a5d92ff135b58de96741dda78b1009a2181cf8b45d2f72": "中文", + "122802ca08e39c2ef46f6a81379dc5683bd8aa074dfb54259f0add4d8b5504bc": "aécio", + "1032308151c0f4f5f8d4e0d96956352eb8ff87da98df8878d8795a858a7e7c08": "𠜎" + }, + "UTF8 more than 64 bytes": { + "d32a41d9858e45b68402f77cf9f3c3f992c36a4bffd230f78d666c87f97eaf7e": "訊息摘要演算法第五版（英語：Message-Digest Algorithm 5，縮寫為MD5），是當前電腦領域用於確保資訊傳輸完整一致而廣泛使用的雜湊演算法之一", + "bd1abad59e6b8ad69bc17b6e05aa13f0cb725467fbeb45b83d3e4094332d1367": "訊息摘要演算法第五版（英語：Message-Digest Algorithm 5，縮寫為MD5），是當前電腦領域用於確保資訊傳輸完整一致而廣泛使用的雜湊演算法之一（又譯雜湊演算法、摘要演算法等），主流程式語言普遍已有MD5的實作。" + }, + "special length": { + "99fb09c8564fbd52274cfaf1130ae02dad89efac9a31dc00e9bfc13db1ff4f56": "0123456780123456780123456780123456780123456780123456780", + "7a3204b58878f5a65a54f77e270d5df579a8016e0e472cc91833689c4cf8ca07": "01234567801234567801234567801234567801234567801234567801", + "f4aa5f7692e6fee7237510b9a886f7b7aa4098926b45eaf70672bdd6d316a633": "0123456780123456780123456780123456780123456780123456780123456780", + "3f8fc8ec35656592ce61bf44895b6d94077aae3bddd99236a0b04ccf936699ed": "01234567801234567801234567801234567801234567801234567801234567801234567", + "4cb330a62170d92fe3d03bcf9284b590cf08d38d3a3c1e661abba3641d0b7502": "012345678012345678012345678012345678012345678012345678012345678012345678" + }, + "Array": { + "c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a": [], + "547cf572033bb67ae341d010b348691ee9c550d07b796e0c6e6ad3503fa36cb3": [211, 212], + "dd9d67b371519c339ed8dbd25af90e976a1eeefd4ad3d889005e532fc5bef04d": [84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103], + "3f8fc8ec35656592ce61bf44895b6d94077aae3bddd99236a0b04ccf936699ed": [48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55] + }, + "Uint8Array": { + "547cf572033bb67ae341d010b348691ee9c550d07b796e0c6e6ad3503fa36cb3": new Uint8Array([211, 212]), + "dd9d67b371519c339ed8dbd25af90e976a1eeefd4ad3d889005e532fc5bef04d": new Uint8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) + }, + "Int8Array": { + "dd9d67b371519c339ed8dbd25af90e976a1eeefd4ad3d889005e532fc5bef04d": new Int8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) + }, + "ArrayBuffer": { + "c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a": new ArrayBuffer(0), + "10baad1713566ac2333467bddb0597dec9066120dd72ac2dcb8394221dcbe43d": new ArrayBuffer(1) + } + }, + sha512: { + "ascii": { + "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": "", + "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6": "The quick brown fox jumps over the lazy dog", + "91ea1245f20d46ae9a037a989f54f1f790f0a47607eeb8a14d12890cea77a1bbc6c7ed9cf205e67b7f2b8fd4c7dfd3a7a8617e45f3c463d481c7e586c39ac1ed": "The quick brown fox jumps over the lazy dog." + }, + "ascii more than 64 bytes": { + "a8dedff31e3be9df6413ef5b4ecb93d62d3fbcb04297552eab5370e04afd45927854a4373037e81a50186e678d818c9ba824f4c850f3d0f02764af0252076979": "The MD5 message-digest algorithm is a widely used cryptographic hash function producing a 128-bit (16-byte) hash value, typically expressed in text format as a 32 digit hexadecimal number. MD5 has been utilized in a wide variety of cryptographic applications, and is also commonly used to verify data integrity." + }, + "UTF8": { + "8b88efc2ebbcbdad5ac2d65af05bec57bda25e71fd5fb25bbd892057a2755fbd05d8d8491cb2946febd5b0f124ffdfbaecf7e34946353c4f1b5ab29545895468": "中文", + "e1c6925243db76985abacaf9fa85e22697f549e67f65a36c88e4046a2260990ff9eefc3402396ea8dcbe8c592d8d5671bea612156eda38d3708d394bbd17d493": "aécio", + "f3e7ee9cdf7dbb52f7edd59ce3d49868c64f2b3aceceab060b8eaaebdf9de0dae5866d660e3319c5aad426a2176cb1703efc73eb24d1a90458ceda1b7f4e3940": "𠜎" + }, + "UTF8 more than 64 bytes": { + "6cb7f6d3381a187edadb43c7cdcfbbed4d2c213a7dce8ea08fe42b9882b64e643202b4974a6db94f94650ab9173d97c58bd59f6d19d27e01aab76d8d08855c65": "訊息摘要演算法第五版（英語：Message-Digest Algorithm 5，縮寫為MD5），是當前電腦領域用於確保資訊傳輸完整一致而廣泛使用的雜湊演算法之一", + "d24af1901aaf1458f089a6eddf784ce61c3012aee0df98bdb67ad2dc6b41a3b4051d40caac524373930ae396a2dde99a9204871b40892eea3e5f3c8d46da0c3c": "訊息摘要演算法第五版（英語：Message-Digest Algorithm 5，縮寫為MD5），是當前電腦領域用於確保資訊傳輸完整一致而廣泛使用的雜湊演算法之一（又譯雜湊演算法、摘要演算法等），主流程式語言普遍已有MD5的實作。" + }, + "special length": { + "6b4a72eb22d2d24c0a429dd99ce5835b134144ac5fce446f66dbf2f421dcc5f8a177e4774f4a48173c5640724b186c2c4112a80937b1167f3e7bb511f4c41b6a": "0123456780123456780123456780123456780123456780123456780", + "76f3cb2ed5b0b405479495b2d3576f4b469b6ffc4b06e3b512a658b84c1b91cf72c41c54d8714ecf19d04696f09e0034632fe98ae848ffd35b83c7e72399a590": "01234567801234567801234567801234567801234567801234567801", + "56d2391faebd8d69b067cd5c0cb364ffc2e2ab87ce5bb06a562b44c8dcb0b83816ad2c0c062537838992b181fadc43ff00e1ebb92ddb1129b81b4864bafb5f63": "0123456780123456780123456780123456780123456780123456780123456780", + "317ab88f192258711b8ae0197395b7a8191796fb41140c16c596699481149b47130e26b3bfa724227202fa8371752ca92e3cb9dd202caf29334038e0848cb43f": "01234567801234567801234567801234567801234567801234567801234567801234567", + "23880e96199df52b4386d190adddaa33cbf7e0bfa7d2067c60eb44ee103667fd002c32e184195fef65fd4178853b1c661d9f260d721df85872e5f645f4388841": "012345678012345678012345678012345678012345678012345678012345678012345678" + }, + "Array": { + "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": [], + "8df0195b2807fdc8c7674c191562e9d0db38b257cc0d3df64669878fe5bb1bbaff53cc8898edcf46cbecb945dc71b6ad738da8ca6f3a824123a54afde5d1d5b0": [211, 212], + "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6": [84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103], + "317ab88f192258711b8ae0197395b7a8191796fb41140c16c596699481149b47130e26b3bfa724227202fa8371752ca92e3cb9dd202caf29334038e0848cb43f": [48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55, 56, 48, 49, 50, 51, 52, 53, 54, 55] + }, + "Uint8Array": { + "8df0195b2807fdc8c7674c191562e9d0db38b257cc0d3df64669878fe5bb1bbaff53cc8898edcf46cbecb945dc71b6ad738da8ca6f3a824123a54afde5d1d5b0": new Uint8Array([211, 212]), + "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6": new Uint8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) + }, + "Int8Array": { + "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6": new Int8Array([84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 115, 32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100, 111, 103]) + }, + "ArrayBuffer": { + "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": new ArrayBuffer(0), + "b8244d028981d693af7b456af8efa4cad63d282e19ff14942c246e50d9351d22704a802a71c3580b6370de4ceb293c324a8423342557d4e5c38438f0e36910ee": new ArrayBuffer(1) + } + }, + hmacSha512bits224: { + "Test Vectors": { + "b244ba01307c0e7a8ccaad13b1067a4cf6b961fe0c6a20bda3d92039": [ + [0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b], + "Hi There" + ], + "4a530b31a79ebcce36916546317c45f247d83241dfb818fd37254bde": [ + "Jefe", + "what do ya want for nothing?" + ], + "db34ea525c2c216ee5a6ccb6608bea870bbef12fd9b96a5109e2b6fc": [ + [0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa], + [0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd] + ], + "c2391863cda465c6828af06ac5d4b72d0b792109952da530e11a0d26": [ + [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19], + [0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd] + ], + "29bef8ce88b54d4226c3c7718ea9e32ace2429026f089e38cea9aeda": [ + [0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa], + "Test Using Larger Than Block-Size Key - Hash Key First" + ], + "82a9619b47af0cea73a8b9741355ce902d807ad87ee9078522a246e1": [ + [0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa], + "This is a test using a larger than block-size key and a larger than block-size data. The key needs to be hashed before being used by the HMAC algorithm." + ] + }, + "UTF8": { + "24e1153464bf5ec62ad2eeeb88ff644f2441a124d1e16e8ae5fb1508": ["中文", "aécio"], + "7a08cecb4700304bc5c466acc1fb312d198374817052a03df07610c6": ["aécio", "𠜎"], + "697973678b7d0075676ec3cbbc19e343ed16fa20c14d8074b76b0861": ["𠜎", "中文"] + }, + "Uint8Array": { + "defdc4a1a6597147ea0c7d0a59ae0a5e64b9413a6400acac28aecdd1": [new Uint8Array(0), "Hi There"] + }, + "ArrayBuffer": { + "defdc4a1a6597147ea0c7d0a59ae0a5e64b9413a6400acac28aecdd1": [new ArrayBuffer(0), "Hi There"] + } + }, + hmacSha512bits256: { + "Test Vectors": { + "9f9126c3d9c3c330d760425ca8a217e31feae31bfe70196ff81642b868402eab": [ + [0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b], + "Hi There" + ], + "6df7b24630d5ccb2ee335407081a87188c221489768fa2020513b2d593359456": [ + "Jefe", + "what do ya want for nothing?" + ], + "229006391d66c8ecddf43ba5cf8f83530ef221a4e9401840d1bead5137c8a2ea": [ + [0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa], + [0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd] + ], + "36d60c8aa1d0be856e10804cf836e821e8733cbafeae87630589fd0b9b0a2f4c": [ + [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19], + [0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd] + ], + "87123c45f7c537a404f8f47cdbedda1fc9bec60eeb971982ce7ef10e774e6539": [ + [0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa], + "Test Using Larger Than Block-Size Key - Hash Key First" + ], + "6ea83f8e7315072c0bdaa33b93a26fc1659974637a9db8a887d06c05a7f35a66": [ + [0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa], + "This is a test using a larger than block-size key and a larger than block-size data. The key needs to be hashed before being used by the HMAC algorithm." + ] + }, + "UTF8": { + "633400fa4bc12c3690efa218c90b56ab1af81b91ad62b57bdbe84988c51071e0": ["中文", "aécio"], + "80eff00e32e0c0813d4c04e296b5ac079ec896e673cc04b0ff14222e151ad0b0": ["aécio", "𠜎"], + "3f801c729e5330a0b91aecc751a26c35688a94989e2098c73bf0c6ac02b99e58": ["𠜎", "中文"] + }, + "Uint8Array": { + "1e08e33f9357abd2a3cfbc82a623d892bb6dccf175d22c0cf24269a7a59dfad6": [new Uint8Array(0), "Hi There"] + }, + "ArrayBuffer": { + "1e08e33f9357abd2a3cfbc82a623d892bb6dccf175d22c0cf24269a7a59dfad6": [new ArrayBuffer(0), "Hi There"] + } + }, + hmacSha512: { + "Test Vectors": { + "87aa7cdea5ef619d4ff0b4241a1d6cb02379f4e2ce4ec2787ad0b30545e17cdedaa833b7d6b8a702038b274eaea3f4e4be9d914eeb61f1702e696c203a126854": [ + [0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b], + "Hi There" + ], + "164b7a7bfcf819e2e395fbe73b56e0a387bd64222e831fd610270cd7ea2505549758bf75c05a994a6d034f65f8f0e6fdcaeab1a34d4a6b4b636e070a38bce737": [ + "Jefe", + "what do ya want for nothing?" + ], + "fa73b0089d56a284efb0f0756c890be9b1b5dbdd8ee81a3655f83e33b2279d39bf3e848279a722c806b485a47e67c807b946a337bee8942674278859e13292fb": [ + [0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa], + [0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd] + ], + "b0ba465637458c6990e5a8c5f61d4af7e576d97ff94b872de76f8050361ee3dba91ca5c11aa25eb4d679275cc5788063a5f19741120c4f2de2adebeb10a298dd": [ + [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19], + [0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd] + ], + "80b24263c7c1a3ebb71493c1dd7be8b49b46d1f41b4aeec1121b013783f8f3526b56d037e05f2598bd0fd2215d6a1e5295e64f73f63f0aec8b915a985d786598": [ + [0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa], + "Test Using Larger Than Block-Size Key - Hash Key First" + ], + "e37b6a775dc87dbaa4dfa9f96e5e3ffddebd71f8867289865df5a32d20cdc944b6022cac3c4982b10d5eeb55c3e4de15134676fb6de0446065c97440fa8c6a58": [ + [0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa], + "This is a test using a larger than block-size key and a larger than block-size data. The key needs to be hashed before being used by the HMAC algorithm." + ] + }, + "UTF8": { + "e9e5906be0aecbc028a5fc759c9dbb86efc9a22950af8e678302a215aeee0b021edc50bbdd71c656730177b7e96c9a3bcf3cb9592bc84a5f3e8900cb67c7eca6": ["中文", "aécio"], + "d02a8d258d855967d5be47240bbedd986a31c29eb5beb35abdbe2725651bf33a195cdfaadb9e76dc4790c71dfea33f708afa04b9471d03f5f0db8440993b9612": ["aécio", "𠜎"], + "a443d463546586a5dd591ef848f0939c3a7089d63ef81d58ccc0a2611a1d374a39717d6893ea10d61ca0e87d5be7c80b29b2ed991c4a62e12d10c7f6b1b9d7ae": ["𠜎", "中文"] + }, + "Uint8Array": { + "f7688a104326d36c1940f6d28d746c0661d383e0d14fe8a04649444777610f5dd9565a36846ab9e9e734cf380d3a070d8ef021b5f3a50c481710a464968e3419": [new Uint8Array(0), "Hi There"] + }, + "ArrayBuffer": { + "f7688a104326d36c1940f6d28d746c0661d383e0d14fe8a04649444777610f5dd9565a36846ab9e9e734cf380d3a070d8ef021b5f3a50c481710a464968e3419": [new ArrayBuffer(0), "Hi There"] + } + }, +}; + +const methods = ["array", "arrayBuffer", "digest", "hex"] as const; + +for (const method of methods) { + for (const [name, tests] of Object.entries(fixtures.sha512bits224)) { + let i = 1; + for (const [expected, message] of Object.entries(tests)) { + test({ + name: `sha512/224.${method}() - ${name} - #${i++}`, + fn() { + const algorithm = new Sha512(224); + algorithm.update(message); + const actual = + method === "hex" + ? algorithm[method]() + : toHexString(algorithm[method]()); + assertEquals(actual, expected); + }, + }); + } + } +} + +for (const method of methods) { + for (const [name, tests] of Object.entries(fixtures.sha512bits256)) { + let i = 1; + for (const [expected, message] of Object.entries(tests)) { + test({ + name: `sha512/256.${method}() - ${name} - #${i++}`, + fn() { + const algorithm = new Sha512(256); + algorithm.update(message); + const actual = + method === "hex" + ? algorithm[method]() + : toHexString(algorithm[method]()); + assertEquals(actual, expected); + }, + }); + } + } +} + +for (const method of methods) { + for (const [name, tests] of Object.entries(fixtures.sha512)) { + let i = 1; + for (const [expected, message] of Object.entries(tests)) { + test({ + name: `sha512.${method}() - ${name} - #${i++}`, + fn() { + const algorithm = new Sha512(); + algorithm.update(message); + const actual = + method === "hex" + ? algorithm[method]() + : toHexString(algorithm[method]()); + assertEquals(actual, expected); + }, + }); + } + } +} + +for (const method of methods) { + for (const [name, tests] of Object.entries(fixtures.hmacSha512bits224)) { + let i = 1; + for (const [expected, [key, message]] of Object.entries(tests)) { + test({ + name: `hmacSha512/224.${method}() - ${name} - #${i++}`, + fn() { + const algorithm = new HmacSha512(key, 224); + algorithm.update(message); + const actual = + method === "hex" + ? algorithm[method]() + : toHexString(algorithm[method]()); + assertEquals(actual, expected); + }, + }); + } + } +} + +for (const method of methods) { + for (const [name, tests] of Object.entries(fixtures.hmacSha512bits256)) { + let i = 1; + for (const [expected, [key, message]] of Object.entries(tests)) { + test({ + name: `hmacSha512/256.${method}() - ${name} - #${i++}`, + fn() { + const algorithm = new HmacSha512(key, 256); + algorithm.update(message); + const actual = + method === "hex" + ? algorithm[method]() + : toHexString(algorithm[method]()); + assertEquals(actual, expected); + }, + }); + } + } +} + +for (const method of methods) { + for (const [name, tests] of Object.entries(fixtures.hmacSha512)) { + let i = 1; + for (const [expected, [key, message]] of Object.entries(tests)) { + test({ + name: `hmacSha512.${method}() - ${name} - #${i++}`, + fn() { + const algorithm = new HmacSha512(key); + algorithm.update(message); + const actual = + method === "hex" + ? algorithm[method]() + : toHexString(algorithm[method]()); + assertEquals(actual, expected); + }, + }); + } + } +} + +test("[hash/sha512] test Uint8Array from Reader", async () => { + const data = await Deno.readFile(join(testdataDir, "hashtest")); + const hash = new Sha512().update(data).hex(); + assertEquals( + hash, + "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff" + ); +}); diff --git a/std/http/README.md b/std/http/README.md index 4e30a3a56542a3..0f97fd0c0e88b6 100644 --- a/std/http/README.md +++ b/std/http/README.md @@ -2,9 +2,9 @@ ```typescript import { serve } from "https://deno.land/std/http/server.ts"; -const s = serve({ port: 8000 }); +const server = serve({ port: 8000 }); console.log("http://localhost:8000/"); -for await (const req of s) { +for await (const req of server) { req.respond({ body: "Hello World\n" }); } ``` diff --git a/std/http/_io.ts b/std/http/_io.ts index 631adafd01f6b4..1db0fc29245437 100644 --- a/std/http/_io.ts +++ b/std/http/_io.ts @@ -1,6 +1,6 @@ import { BufReader, BufWriter } from "../io/bufio.ts"; import { TextProtoReader } from "../textproto/mod.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; import { encoder } from "../encoding/utf8.ts"; import { ServerRequest, Response } from "./server.ts"; import { STATUS_TEXT } from "./http_status.ts"; @@ -113,50 +113,60 @@ export function chunkedBodyReader(h: Headers, r: BufReader): Deno.Reader { return { read }; } -const kProhibitedTrailerHeaders = [ - "transfer-encoding", - "content-length", - "trailer", -]; +function isProhibidedForTrailer(key: string): boolean { + const s = new Set(["transfer-encoding", "content-length", "trailer"]); + return s.has(key.toLowerCase()); +} -/** - * Read trailer headers from reader and append values to headers. - * "trailer" field will be deleted. - * */ +/** Read trailer headers from reader and append values to headers. "trailer" + * field will be deleted. */ export async function readTrailers( headers: Headers, r: BufReader ): Promise { - const keys = parseTrailer(headers.get("trailer")); - if (!keys) return; + const trailers = parseTrailer(headers.get("trailer")); + if (trailers == null) return; + const trailerNames = [...trailers.keys()]; const tp = new TextProtoReader(r); const result = await tp.readMIMEHeader(); - assert(result !== null, "trailer must be set"); + if (result == null) { + throw new Deno.errors.InvalidData("Missing trailer header."); + } + const undeclared = [...result.keys()].filter( + (k) => !trailerNames.includes(k) + ); + if (undeclared.length > 0) { + throw new Deno.errors.InvalidData( + `Undeclared trailers: ${Deno.inspect(undeclared)}.` + ); + } for (const [k, v] of result) { - if (!keys.has(k)) { - throw new Error("Undeclared trailer field"); - } - keys.delete(k); headers.append(k, v); } - assert(keys.size === 0, "Missing trailers"); + const missingTrailers = trailerNames.filter((k) => !result.has(k)); + if (missingTrailers.length > 0) { + throw new Deno.errors.InvalidData( + `Missing trailers: ${Deno.inspect(missingTrailers)}.` + ); + } headers.delete("trailer"); } -function parseTrailer(field: string | null): Set | undefined { +function parseTrailer(field: string | null): Headers | undefined { if (field == null) { return undefined; } - const keys = field.split(",").map((v) => v.trim()); - if (keys.length === 0) { - throw new Error("Empty trailer"); + const trailerNames = field.split(",").map((v) => v.trim().toLowerCase()); + if (trailerNames.length === 0) { + throw new Deno.errors.InvalidData("Empty trailer header."); } - for (const invalid of kProhibitedTrailerHeaders) { - if (keys.includes(invalid)) { - throw new Error(`Prohibited field for trailer`); - } + const prohibited = trailerNames.filter((k) => isProhibidedForTrailer(k)); + if (prohibited.length > 0) { + throw new Deno.errors.InvalidData( + `Prohibited trailer names: ${Deno.inspect(prohibited)}.` + ); } - return new Set(keys); + return new Headers(trailerNames.map((key) => [key, ""])); } export async function writeChunkedBody( @@ -177,7 +187,8 @@ export async function writeChunkedBody( await writer.write(endChunk); } -/** write trailer headers to writer. it mostly should be called after writeResponse */ +/** Write trailer headers to writer. It should mostly should be called after + * `writeResponse()`. */ export async function writeTrailers( w: Deno.Writer, headers: Headers, @@ -185,29 +196,31 @@ export async function writeTrailers( ): Promise { const trailer = headers.get("trailer"); if (trailer === null) { - throw new Error('response headers must have "trailer" header field'); + throw new TypeError("Missing trailer header."); } const transferEncoding = headers.get("transfer-encoding"); if (transferEncoding === null || !transferEncoding.match(/^chunked/)) { - throw new Error( - `trailer headers is only allowed for "transfer-encoding: chunked": got "${transferEncoding}"` + throw new TypeError( + `Trailers are only allowed for "transfer-encoding: chunked", got "transfer-encoding: ${transferEncoding}".` ); } const writer = BufWriter.create(w); - const trailerHeaderFields = trailer - .split(",") - .map((s) => s.trim().toLowerCase()); - for (const f of trailerHeaderFields) { - assert( - !kProhibitedTrailerHeaders.includes(f), - `"${f}" is prohibited for trailer header` + const trailerNames = trailer.split(",").map((s) => s.trim().toLowerCase()); + const prohibitedTrailers = trailerNames.filter((k) => + isProhibidedForTrailer(k) + ); + if (prohibitedTrailers.length > 0) { + throw new TypeError( + `Prohibited trailer names: ${Deno.inspect(prohibitedTrailers)}.` ); } + const undeclared = [...trailers.keys()].filter( + (k) => !trailerNames.includes(k) + ); + if (undeclared.length > 0) { + throw new TypeError(`Undeclared trailers: ${Deno.inspect(undeclared)}.`); + } for (const [key, value] of trailers) { - assert( - trailerHeaderFields.includes(key), - `Not trailer header field: ${key}` - ); await writer.write(encoder.encode(`${key}: ${value}\r\n`)); } await writer.write(encoder.encode("\r\n")); diff --git a/std/http/_io_test.ts b/std/http/_io_test.ts index c22ebdf07e4d74..3b385d013d3b0c 100644 --- a/std/http/_io_test.ts +++ b/std/http/_io_test.ts @@ -1,5 +1,4 @@ import { - AssertionError, assertThrowsAsync, assertEquals, assert, @@ -82,7 +81,7 @@ test("chunkedBodyReader with trailers", async () => { test("readTrailers", async () => { const h = new Headers({ - trailer: "deno,node", + trailer: "Deno, Node", }); const trailer = ["deno: land", "node: js", "", ""].join("\r\n"); await readTrailers(h, new BufReader(new Buffer(encode(trailer)))); @@ -105,14 +104,14 @@ test("readTrailer should throw if undeclared headers found in trailer", async () async () => { await readTrailers(h, new BufReader(new Buffer(encode(trailer)))); }, - Error, - "Undeclared trailer field" + Deno.errors.InvalidData, + `Undeclared trailers: [ "` ); } }); test("readTrailer should throw if trailer contains prohibited fields", async () => { - for (const f of ["content-length", "trailer", "transfer-encoding"]) { + for (const f of ["Content-Length", "Trailer", "Transfer-Encoding"]) { const h = new Headers({ trailer: f, }); @@ -120,8 +119,8 @@ test("readTrailer should throw if trailer contains prohibited fields", async () async () => { await readTrailers(h, new BufReader(new Buffer())); }, - Error, - "Prohibited field for trailer" + Deno.errors.InvalidData, + `Prohibited trailer names: [ "` ); } }); @@ -145,15 +144,15 @@ test("writeTrailer should throw", async () => { () => { return writeTrailers(w, new Headers(), new Headers()); }, - Error, - 'must have "trailer"' + TypeError, + "Missing trailer header." ); await assertThrowsAsync( () => { return writeTrailers(w, new Headers({ trailer: "deno" }), new Headers()); }, - Error, - "only allowed" + TypeError, + `Trailers are only allowed for "transfer-encoding: chunked", got "transfer-encoding: null".` ); for (const f of ["content-length", "trailer", "transfer-encoding"]) { await assertThrowsAsync( @@ -164,8 +163,8 @@ test("writeTrailer should throw", async () => { new Headers({ [f]: "1" }) ); }, - AssertionError, - "prohibited" + TypeError, + `Prohibited trailer names: [ "` ); } await assertThrowsAsync( @@ -176,8 +175,8 @@ test("writeTrailer should throw", async () => { new Headers({ node: "js" }) ); }, - AssertionError, - "Not trailer" + TypeError, + `Undeclared trailers: [ "node" ].` ); }); diff --git a/std/http/cookie.ts b/std/http/cookie.ts index 521cb6ded5e90e..88a71626c91c64 100644 --- a/std/http/cookie.ts +++ b/std/http/cookie.ts @@ -2,7 +2,7 @@ // Structured similarly to Go's cookie.go // https://github.com/golang/go/blob/master/src/net/http/cookie.go import { ServerRequest, Response } from "./server.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; import { toIMF } from "../datetime/mod.ts"; export interface Cookies { @@ -130,9 +130,9 @@ export function setCookie(res: Response, cookie: Cookie): void { * @param name Name of the cookie to Delete * Example: * - * delCookie(res,'foo'); + * deleteCookie(res,'foo'); */ -export function delCookie(res: Response, name: string): void { +export function deleteCookie(res: Response, name: string): void { setCookie(res, { name: name, value: "", diff --git a/std/http/cookie_test.ts b/std/http/cookie_test.ts index 2c8543053515a4..e221b3363b6b7e 100644 --- a/std/http/cookie_test.ts +++ b/std/http/cookie_test.ts @@ -1,6 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import { ServerRequest, Response } from "./server.ts"; -import { getCookies, delCookie, setCookie } from "./cookie.ts"; +import { getCookies, deleteCookie, setCookie } from "./cookie.ts"; import { assert, assertEquals } from "../testing/asserts.ts"; const { test } = Deno; @@ -36,7 +36,7 @@ test({ name: "Cookie Delete", fn(): void { const res: Response = {}; - delCookie(res, "deno"); + deleteCookie(res, "deno"); assertEquals( res.headers?.get("Set-Cookie"), "deno=; Expires=Thu, 01 Jan 1970 00:00:00 GMT" diff --git a/std/http/file_server.ts b/std/http/file_server.ts index 81515b04b832f7..66babfd5703068 100755 --- a/std/http/file_server.ts +++ b/std/http/file_server.ts @@ -1,4 +1,4 @@ -#!/usr/bin/env -S deno --allow-net +#!/usr/bin/env -S deno run --allow-net --allow-read // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. // This program serves files in the current directory over HTTP. @@ -10,7 +10,7 @@ const { args, stat, readDir, open, exit } = Deno; import { posix, extname } from "../path/mod.ts"; import { listenAndServe, ServerRequest, Response } from "./server.ts"; import { parse } from "../flags/mod.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; interface EntryInfo { mode: string; @@ -34,10 +34,7 @@ interface FileServerArgs { const encoder = new TextEncoder(); const serverArgs = parse(args) as FileServerArgs; - -const CORSEnabled = serverArgs.cors ? true : false; const target = posix.resolve(serverArgs._[0] ?? ""); -const addr = `0.0.0.0:${serverArgs.port ?? serverArgs.p ?? 4507}`; const MEDIA_TYPES: Record = { ".md": "text/markdown", @@ -52,6 +49,7 @@ const MEDIA_TYPES: Record = { ".jsx": "text/jsx", ".gz": "application/gzip", ".css": "text/css", + ".wasm": "application/wasm", }; /** Returns the content-type based on the extension of a path. */ @@ -59,23 +57,6 @@ function contentType(path: string): string | undefined { return MEDIA_TYPES[extname(path)]; } -if (serverArgs.h ?? serverArgs.help) { - console.log(`Deno File Server - Serves a local directory in HTTP. - -INSTALL: - deno install --allow-net --allow-read https://deno.land/std/http/file_server.ts - -USAGE: - file_server [path] [options] - -OPTIONS: - -h, --help Prints help information - -p, --port Set port - --cors Enable CORS via the "Access-Control-Allow-Origin" header`); - exit(); -} - function modeToString(isDir: boolean, maybeMode: number | null): string { const modeMap = ["---", "--x", "-w-", "-wx", "r--", "r-x", "rw-", "rwx"]; @@ -126,6 +107,9 @@ export async function serveFile( if (contentTypeValue) { headers.set("content-type", contentTypeValue); } + req.done.then(() => { + file.close(); + }); return { status: 200, body: file, @@ -315,6 +299,26 @@ function html(strings: TemplateStringsArray, ...values: unknown[]): string { } function main(): void { + const CORSEnabled = serverArgs.cors ? true : false; + const addr = `0.0.0.0:${serverArgs.port ?? serverArgs.p ?? 4507}`; + + if (serverArgs.h ?? serverArgs.help) { + console.log(`Deno File Server + Serves a local directory in HTTP. + + INSTALL: + deno install --allow-net --allow-read https://deno.land/std/http/file_server.ts + + USAGE: + file_server [path] [options] + + OPTIONS: + -h, --help Prints help information + -p, --port Set port + --cors Enable CORS via the "Access-Control-Allow-Origin" header`); + exit(); + } + listenAndServe( addr, async (req): Promise => { diff --git a/std/http/file_server_test.ts b/std/http/file_server_test.ts index 97c07c2503913c..dbbaf81ff799b7 100644 --- a/std/http/file_server_test.ts +++ b/std/http/file_server_test.ts @@ -38,6 +38,24 @@ async function startFileServer({ assert(s !== null && s.includes("server listening")); } +async function startFileServerAsLibrary({}: FileServerCfg = {}): Promise { + fileServer = await Deno.run({ + cmd: [ + Deno.execPath(), + "run", + "--allow-read", + "--allow-net", + "http/testdata/file_server_as_library.ts", + ], + stdout: "piped", + stderr: "null", + }); + assert(fileServer.stdout != null); + const r = new TextProtoReader(new BufReader(fileServer.stdout)); + const s = await r.readLine(); + assert(s !== null && s.includes("Server running...")); +} + async function killFileServer(): Promise { fileServer.close(); // Process.close() kills the file server process. However this termination @@ -166,3 +184,13 @@ test("contentType", async () => { assertEquals(contentType, "text/html"); (response.body as Deno.File).close(); }); + +test("file_server running as library", async function (): Promise { + await startFileServerAsLibrary(); + try { + const res = await fetch("http://localhost:8000"); + assertEquals(res.status, 200); + } finally { + await killFileServer(); + } +}); diff --git a/std/http/http_status.ts b/std/http/http_status.ts index ce4338705c1411..a0d0c278dc4bbb 100644 --- a/std/http/http_status.ts +++ b/std/http/http_status.ts @@ -8,7 +8,8 @@ export enum Status { SwitchingProtocols = 101, /** RFC 2518, 10.1 */ Processing = 102, - + /** RFC 8297 **/ + EarlyHints = 103, /** RFC 7231, 6.3.1 */ OK = 200, /** RFC 7231, 6.3.2 */ @@ -93,6 +94,8 @@ export enum Status { Locked = 423, /** RFC 4918, 11.4 */ FailedDependency = 424, + /** RFC 8470, 5.2 */ + TooEarly = 425, /** RFC 7231, 6.5.15 */ UpgradeRequired = 426, /** RFC 6585, 3 */ @@ -132,6 +135,7 @@ export const STATUS_TEXT = new Map([ [Status.Continue, "Continue"], [Status.SwitchingProtocols, "Switching Protocols"], [Status.Processing, "Processing"], + [Status.EarlyHints, "Early Hints"], [Status.OK, "OK"], [Status.Created, "Created"], [Status.Accepted, "Accepted"], @@ -173,6 +177,7 @@ export const STATUS_TEXT = new Map([ [Status.UnprocessableEntity, "Unprocessable Entity"], [Status.Locked, "Locked"], [Status.FailedDependency, "Failed Dependency"], + [Status.TooEarly, "Too Early"], [Status.UpgradeRequired, "Upgrade Required"], [Status.PreconditionRequired, "Precondition Required"], [Status.TooManyRequests, "Too Many Requests"], diff --git a/std/http/server.ts b/std/http/server.ts index a372b39a50c255..18bfc4731d5b63 100644 --- a/std/http/server.ts +++ b/std/http/server.ts @@ -1,7 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import { encode } from "../encoding/utf8.ts"; import { BufReader, BufWriter } from "../io/bufio.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; import { deferred, Deferred, MuxAsyncIterator } from "../async/mod.ts"; import { bodyReader, @@ -53,18 +53,9 @@ export class ServerRequest { private _body: Deno.Reader | null = null; /** - * Body of the request. + * Body of the request. The easiest way to consume the body is: * - * const buf = new Uint8Array(req.contentLength); - * let bufSlice = buf; - * let totRead = 0; - * while (true) { - * const nread = await req.body.read(bufSlice); - * if (nread === null) break; - * totRead += nread; - * if (totRead >= req.contentLength) break; - * bufSlice = bufSlice.subarray(nread); - * } + * const buf: Uint8Array = await Deno.readAll(req.body); */ get body(): Deno.Reader { if (!this._body) { @@ -220,8 +211,12 @@ export class Server implements AsyncIterable { try { conn = await this.listener.accept(); } catch (error) { - if (error instanceof Deno.errors.BadResource) { - return; + if ( + error instanceof Deno.errors.BadResource || + error instanceof Deno.errors.InvalidData || + error instanceof Deno.errors.UnexpectedEof + ) { + return mux.add(this.acceptConnAndIterateHttpRequests(mux)); } throw error; } @@ -247,8 +242,8 @@ export type HTTPOptions = Omit; * * import { serve } from "https://deno.land/std/http/server.ts"; * const body = "Hello World\n"; - * const s = serve({ port: 8000 }); - * for await (const req of s) { + * const server = serve({ port: 8000 }); + * for await (const req of server) { * req.respond({ body }); * } */ diff --git a/std/http/server_test.ts b/std/http/server_test.ts index 0560f7f8daab56..93b117059b0e9c 100644 --- a/std/http/server_test.ts +++ b/std/http/server_test.ts @@ -10,7 +10,7 @@ import { assert, assertEquals, assertMatch, - assertStrContains, + assertStringContains, assertThrowsAsync, } from "../testing/asserts.ts"; import { Response, ServerRequest, Server, serve } from "./server.ts"; @@ -480,7 +480,7 @@ test({ const nread = await conn.read(res); assert(nread !== null); const resStr = new TextDecoder().decode(res.subarray(0, nread)); - assertStrContains(resStr, "/hello"); + assertStringContains(resStr, "/hello"); server.close(); await p; // Client connection should still be open, verify that @@ -496,7 +496,6 @@ test({ async fn(): Promise { const serverRoutine = async (): Promise => { const server = serve(":8124"); - // @ts-ignore for await (const req of server) { await assertThrowsAsync(async () => { await req.respond({ @@ -546,3 +545,73 @@ test({ assert((await entry).done); }, }); + +test({ + name: "serveTLS Invalid Cert", + fn: async (): Promise => { + // Runs a simple server as another process + const p = Deno.run({ + cmd: [ + Deno.execPath(), + "run", + "--allow-net", + "--allow-read", + "http/testdata/simple_https_server.ts", + ], + stdout: "piped", + }); + + let serverIsRunning = true; + const statusPromise = p + .status() + .then((): void => { + serverIsRunning = false; + }) + .catch((_): void => {}); // Ignores the error when closing the process. + + try { + const r = new TextProtoReader(new BufReader(p.stdout!)); + const s = await r.readLine(); + assert( + s !== null && s.includes("server listening"), + "server must be started" + ); + // Invalid certificate, connection should throw + // but should not crash the server + assertThrowsAsync( + () => + Deno.connectTls({ + hostname: "localhost", + port: 4503, + // certFile + }), + Deno.errors.InvalidData + ); + + // Valid request after invalid + const conn = await Deno.connectTls({ + hostname: "localhost", + port: 4503, + certFile: "http/testdata/tls/RootCA.pem", + }); + + await Deno.writeAll( + conn, + new TextEncoder().encode("GET / HTTP/1.0\r\n\r\n") + ); + const res = new Uint8Array(100); + const nread = await conn.read(res); + assert(nread !== null); + conn.close(); + const resStr = new TextDecoder().decode(res.subarray(0, nread)); + assert(resStr.includes("Hello HTTPS")); + assert(serverIsRunning); + } finally { + // Stops the sever and allows `p.status()` promise to resolve + Deno.kill(p.pid, Deno.Signal.SIGKILL); + await statusPromise; + p.stdout!.close(); + p.close(); + } + }, +}); diff --git a/std/http/testdata/file_server_as_library.ts b/std/http/testdata/file_server_as_library.ts new file mode 100644 index 00000000000000..ab5fdcaabe5ceb --- /dev/null +++ b/std/http/testdata/file_server_as_library.ts @@ -0,0 +1,12 @@ +import { serve } from "../server.ts"; +import { serveFile } from "../file_server.ts"; + +const server = serve({ port: 8000 }); + +console.log('Server running...'); + +for await (const req of server) { + serveFile(req, './http/testdata/hello.html').then(response => { + req.respond(response); + }); +} \ No newline at end of file diff --git a/std/io/README.md b/std/io/README.md new file mode 100644 index 00000000000000..2432a468b0e189 --- /dev/null +++ b/std/io/README.md @@ -0,0 +1,125 @@ +# std/io + +## Bufio + +**Uses:** + +### readLines + +Read reader[like file], line by line + +```ts title="readLines" +import { readLines } from "https://deno.land/std/io/mod.ts"; +import * as path from "https://deno.land/std/path/mod.ts"; + +const filename = path.join(Deno.cwd(), "std/io/README.md"); +let fileReader = await Deno.open(filename); + +for await (let line of readLines(fileReader)) { + console.log(line); +} +``` + +**Output:** + +````text +# std/io + +## readLines + +```ts +import * as path from "https://deno.land/std/path/mod.ts"; + +## Rest of the file +```` + +### readStringDelim + +Read reader`[like file]` chunk by chunk, splitting based on delimiter. + +```ts title="readStringDelim" +import { readLines } from "https://deno.land/std/io/mod.ts"; +import * as path from "https://deno.land/std/path/mod.ts"; + +const filename = path.join(Deno.cwd(), "std/io/README.md"); +let fileReader = await Deno.open(filename); + +for await (let line of readStringDelim(fileReader, "\n")) { + console.log(line); +} +``` + +**Output:** + +````text +# std/io + +## readLines + +```ts +import * as path from "https://deno.land/std/path/mod.ts"; + +## Rest of the file +```` + +## Reader + +### StringReader + +Create a `Reader` object for `string`. + +```ts +import { StringReader } from "https://deno.land/std/io/mod.ts"; + +const data = new Uint8Array(6); +const r = new StringReader("abcdef"); +const res0 = await r.read(data); +const res1 = await r.read(new Uint8Array(6)); + +// Number of bytes read +console.log(res0); // 6 +console.log(res1); // null, no byte left to read. EOL + +// text + +console.log(new TextDecoder().decode(data)); // abcdef +``` + +**Output:** + +```text +6 +null +abcdef +``` + +## Writer + +### StringWriter + +Create a `Writer` object for `string`. + +```ts +import { + StringWriter, + StringReader, + copyN, +} from "https://deno.land/std/io/mod.ts"; + +const w = new StringWriter("base"); +const r = new StringReader("0123456789"); +await copyN(r, w, 4); // copy 4 bytes + +// Number of bytes read +console.log(w.toString()); //base0123 + +await Deno.copy(r, w); // copy all +console.log(w.toString()); // base0123456789 +``` + +**Output:** + +```text +base0123 +base0123456789 +``` diff --git a/std/io/bufio.ts b/std/io/bufio.ts index 5c005672a392ab..4ebec13e1d983b 100644 --- a/std/io/bufio.ts +++ b/std/io/bufio.ts @@ -7,7 +7,7 @@ type Reader = Deno.Reader; type Writer = Deno.Writer; type WriterSync = Deno.WriterSync; import { charCode, copyBytes } from "./util.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; const DEFAULT_BUF_SIZE = 4096; const MIN_BUF_SIZE = 16; diff --git a/std/io/bufio_test.ts b/std/io/bufio_test.ts index 671ed2115e91cf..b03640d529ab2f 100644 --- a/std/io/bufio_test.ts +++ b/std/io/bufio_test.ts @@ -17,7 +17,8 @@ import { readLines, } from "./bufio.ts"; import * as iotest from "./_iotest.ts"; -import { charCode, copyBytes, stringsReader } from "./util.ts"; +import { StringReader } from "./readers.ts"; +import { charCode, copyBytes } from "./util.ts"; const encoder = new TextEncoder(); @@ -38,7 +39,7 @@ async function readBytes(buf: BufReader): Promise { Deno.test("bufioReaderSimple", async function (): Promise { const data = "hello world"; - const b = new BufReader(stringsReader(data)); + const b = new BufReader(new StringReader(data)); const s = await readBytes(b); assertEquals(s, data); }); @@ -119,7 +120,7 @@ Deno.test("bufioBufReader", async function (): Promise { for (const readmaker of readMakers) { for (const bufreader of bufreaders) { for (const bufsize of bufsizes) { - const read = readmaker.fn(stringsReader(text)); + const read = readmaker.fn(new StringReader(text)); const buf = new BufReader(read, bufsize); const s = await bufreader.fn(buf); const debugStr = @@ -136,7 +137,7 @@ Deno.test("bufioBufferFull", async function (): Promise { const longString = "And now, hello, world! It is the time for all good men to come to the" + " aid of their party"; - const buf = new BufReader(stringsReader(longString), MIN_READ_BUFFER_SIZE); + const buf = new BufReader(new StringReader(longString), MIN_READ_BUFFER_SIZE); const decoder = new TextDecoder(); try { @@ -156,7 +157,7 @@ Deno.test("bufioBufferFull", async function (): Promise { Deno.test("bufioReadString", async function (): Promise { const string = "And now, hello world!"; - const buf = new BufReader(stringsReader(string), MIN_READ_BUFFER_SIZE); + const buf = new BufReader(new StringReader(string), MIN_READ_BUFFER_SIZE); const line = await buf.readString(","); assert(line !== null); @@ -248,7 +249,7 @@ Deno.test("bufioPeek", async function (): Promise { const p = new Uint8Array(10); // string is 16 (minReadBufferSize) long. const buf = new BufReader( - stringsReader("abcdefghijklmnop"), + new StringReader("abcdefghijklmnop"), MIN_READ_BUFFER_SIZE ); @@ -453,7 +454,7 @@ Deno.test( const decoder = new TextDecoder(); const data = "abcdefghijklmnopqrstuvwxyz"; const bufSize = 25; - const b = new BufReader(stringsReader(data), bufSize); + const b = new BufReader(new StringReader(data), bufSize); const r1 = (await b.readLine()) as ReadLineResult; assert(r1 !== null); diff --git a/std/io/ioutil.ts b/std/io/ioutil.ts index aff9f05e2ceb54..3d9e72b569ebe1 100644 --- a/std/io/ioutil.ts +++ b/std/io/ioutil.ts @@ -2,7 +2,7 @@ import { BufReader } from "./bufio.ts"; type Reader = Deno.Reader; type Writer = Deno.Writer; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; const DEFAULT_BUFFER_SIZE = 32 * 1024; diff --git a/std/io/ioutil_test.ts b/std/io/ioutil_test.ts index cf8eaf43739b42..977c7022cbedbb 100644 --- a/std/io/ioutil_test.ts +++ b/std/io/ioutil_test.ts @@ -9,8 +9,9 @@ import { readShort, sliceLongToBytes, } from "./ioutil.ts"; +import { StringReader } from "./readers.ts"; import { BufReader } from "./bufio.ts"; -import { stringsReader, tempFile } from "./util.ts"; +import { tempFile } from "./util.ts"; import * as path from "../path/mod.ts"; class BinaryReader implements Reader { @@ -73,7 +74,7 @@ Deno.test("testSliceLongToBytes2", function (): void { Deno.test("testCopyN1", async function (): Promise { const w = new Buffer(); - const r = stringsReader("abcdefghij"); + const r = new StringReader("abcdefghij"); const n = await copyN(r, w, 3); assertEquals(n, 3); assertEquals(new TextDecoder().decode(w.bytes()), "abc"); @@ -81,7 +82,7 @@ Deno.test("testCopyN1", async function (): Promise { Deno.test("testCopyN2", async function (): Promise { const w = new Buffer(); - const r = stringsReader("abcdefghij"); + const r = new StringReader("abcdefghij"); const n = await copyN(r, w, 11); assertEquals(n, 10); assertEquals(new TextDecoder().decode(w.bytes()), "abcdefghij"); @@ -91,10 +92,17 @@ Deno.test("copyNWriteAllData", async function (): Promise { const { filepath, file } = await tempFile(path.resolve("io")); const size = 16 * 1024 + 1; const data = "a".repeat(32 * 1024); - const r = stringsReader(data); + const r = new StringReader(data); const n = await copyN(r, file, size); // Over max file possible buffer file.close(); await Deno.remove(filepath); assertEquals(n, size); }); + +Deno.test("testStringReaderEof", async function (): Promise { + const r = new StringReader("abc"); + assertEquals(await r.read(new Uint8Array()), 0); + assertEquals(await r.read(new Uint8Array(4)), 3); + assertEquals(await r.read(new Uint8Array(1)), null); +}); diff --git a/std/io/readers.ts b/std/io/readers.ts index 10069986cef510..d4365526362a47 100644 --- a/std/io/readers.ts +++ b/std/io/readers.ts @@ -1,22 +1,18 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +// Based on https://github.com/golang/go/blob/0452f9460f50f0f0aba18df43dc2b31906fb66cc/src/io/io.go +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + type Reader = Deno.Reader; import { encode } from "../encoding/utf8.ts"; +const { Buffer } = Deno; /** Reader utility for strings */ -export class StringReader implements Reader { - private offs = 0; - private buf = new Uint8Array(encode(this.s)); - - constructor(private readonly s: string) {} - - read(p: Uint8Array): Promise { - const n = Math.min(p.byteLength, this.buf.byteLength - this.offs); - p.set(this.buf.slice(this.offs, this.offs + n)); - this.offs += n; - if (n === 0) { - return Promise.resolve(null); - } - return Promise.resolve(n); +export class StringReader extends Buffer { + constructor(private readonly s: string) { + super(encode(s).buffer); } } @@ -40,3 +36,30 @@ export class MultiReader implements Reader { return result; } } + +/** + * A `LimitedReader` reads from `reader` but limits the amount of data returned to just `limit` bytes. + * Each call to `read` updates `limit` to reflect the new amount remaining. + * `read` returns `null` when `limit` <= `0` or + * when the underlying `reader` returns `null`. + */ +export class LimitedReader implements Deno.Reader { + constructor(public reader: Deno.Reader, public limit: number) {} + + async read(p: Uint8Array): Promise { + if (this.limit <= 0) { + return null; + } + + if (p.length > this.limit) { + p = p.subarray(0, this.limit); + } + const n = await this.reader.read(p); + if (n == null) { + return null; + } + + this.limit -= n; + return n; + } +} diff --git a/std/io/readers_test.ts b/std/io/readers_test.ts index b0810f9e0639e0..04e9b7488d35af 100644 --- a/std/io/readers_test.ts +++ b/std/io/readers_test.ts @@ -1,6 +1,6 @@ const { copy, test } = Deno; import { assertEquals } from "../testing/asserts.ts"; -import { MultiReader, StringReader } from "./readers.ts"; +import { LimitedReader, MultiReader, StringReader } from "./readers.ts"; import { StringWriter } from "./writers.ts"; import { copyN } from "./ioutil.ts"; import { decode } from "../encoding/utf8.ts"; @@ -36,3 +36,28 @@ test("ioMultiReader", async function (): Promise { await copy(r, w); assertEquals(w.toString(), "abcdef"); }); + +test("ioLimitedReader", async function (): Promise { + let sr = new StringReader("abc"); + let r = new LimitedReader(sr, 2); + let buffer = await Deno.readAll(r); + assertEquals(decode(buffer), "ab"); + assertEquals(decode(await Deno.readAll(sr)), "c"); + sr = new StringReader("abc"); + r = new LimitedReader(sr, 3); + buffer = await Deno.readAll(r); + assertEquals(decode(buffer), "abc"); + assertEquals((await Deno.readAll(r)).length, 0); + sr = new StringReader("abc"); + r = new LimitedReader(sr, 4); + buffer = await Deno.readAll(r); + assertEquals(decode(buffer), "abc"); + assertEquals((await Deno.readAll(r)).length, 0); +}); + +test("ioLimitedReader", async function (): Promise { + const rb = new StringReader("abc"); + const wb = new StringWriter(); + await Deno.copy(new LimitedReader(rb, -1), wb); + assertEquals(wb.toString(), ""); +}); diff --git a/std/io/util.ts b/std/io/util.ts index 237747a8935fd6..47e48a981d1d27 100644 --- a/std/io/util.ts +++ b/std/io/util.ts @@ -1,9 +1,8 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -const { Buffer, mkdir, open } = Deno; +const { mkdir, open } = Deno; type File = Deno.File; type Reader = Deno.Reader; import * as path from "../path/mod.ts"; -import { encode } from "../encoding/utf8.ts"; /** * Copy bytes from one Uint8Array to another. Bytes from `src` which don't fit @@ -28,10 +27,6 @@ export function charCode(s: string): number { return s.charCodeAt(0); } -export function stringsReader(s: string): Reader { - return new Buffer(encode(s).buffer); -} - /** Create or open a temporal file at specified directory with prefix and * postfix * */ diff --git a/std/log/README.md b/std/log/README.md index 55b2b6cdb319bf..3989d1fb78aac6 100644 --- a/std/log/README.md +++ b/std/log/README.md @@ -107,7 +107,10 @@ interface HandlerOptions { #### `FileHandler` This handler will output to a file using an optional mode (default is `a`, e.g. -append). The file will grow indefinitely. This logger takes `FileOptions`: +append). The file will grow indefinitely. It uses a buffer for writing to file. +Logs can be manually flushed with `fileHandler.flush()`. Log messages with a log +level greater than error are immediately flushed. Logs are also flushed on +process completion. This logger takes `FileOptions`: ```typescript interface FileHandlerOptions { @@ -148,6 +151,11 @@ backups to keep), `log.txt.1` would be renamed to `log.txt.2`, `log.txt` would be renamed to `log.txt.1` and finally `log.txt` would be created from scratch where the new log message would be written. +This handler uses a buffer for writing log messages to file. Logs can be +manually flushed with `fileHandler.flush()`. Log messages with a log level +greater than ERROR are immediately flushed. Logs are also flushed on process +completion. + Options for this handler are: ```typescript diff --git a/std/log/handlers.ts b/std/log/handlers.ts index b22f458ac240c5..5e72ff5826e713 100644 --- a/std/log/handlers.ts +++ b/std/log/handlers.ts @@ -1,5 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -const { open, openSync, close, renameSync, statSync } = Deno; +const { open, openSync, close, renameSync, stat } = Deno; type File = Deno.File; type Writer = Deno.Writer; type OpenOptions = Deno.OpenOptions; @@ -7,6 +7,7 @@ import { getLevelByName, LevelName, LogLevels } from "./levels.ts"; import { LogRecord } from "./logger.ts"; import { red, yellow, blue, bold } from "../fmt/colors.ts"; import { existsSync, exists } from "../fs/exists.ts"; +import { BufWriterSync } from "../io/bufio.ts"; const DEFAULT_FORMATTER = "{levelName} {msg}"; type FormatterFunction = (logRecord: LogRecord) => string; @@ -99,11 +100,13 @@ interface FileHandlerOptions extends HandlerOptions { } export class FileHandler extends WriterHandler { - protected _file!: File; + protected _file: File | undefined; + protected _buf!: BufWriterSync; protected _filename: string; protected _mode: LogMode; protected _openOptions: OpenOptions; - #encoder = new TextEncoder(); + protected _encoder = new TextEncoder(); + #unloadCallback = (): Promise => this.destroy(); constructor(levelName: LevelName, options: FileHandlerOptions) { super(levelName, options); @@ -122,14 +125,35 @@ export class FileHandler extends WriterHandler { async setup(): Promise { this._file = await open(this._filename, this._openOptions); this._writer = this._file; + this._buf = new BufWriterSync(this._file); + + addEventListener("unload", this.#unloadCallback); + } + + handle(logRecord: LogRecord): void { + super.handle(logRecord); + + // Immediately flush if log level is higher than ERROR + if (logRecord.level > LogLevels.ERROR) { + this.flush(); + } } log(msg: string): void { - Deno.writeSync(this._file.rid, this.#encoder.encode(msg + "\n")); + this._buf.writeSync(this._encoder.encode(msg + "\n")); + } + + flush(): void { + if (this._buf?.buffered() > 0) { + this._buf.flush(); + } } destroy(): Promise { - this._file.close(); + this.flush(); + this._file?.close(); + this._file = undefined; + removeEventListener("unload", this.#unloadCallback); return Promise.resolve(); } } @@ -142,6 +166,7 @@ interface RotatingFileHandlerOptions extends FileHandlerOptions { export class RotatingFileHandler extends FileHandler { #maxBytes: number; #maxBackupCount: number; + #currentFileSize = 0; constructor(levelName: LevelName, options: RotatingFileHandlerOptions) { super(levelName, options); @@ -151,9 +176,11 @@ export class RotatingFileHandler extends FileHandler { async setup(): Promise { if (this.#maxBytes < 1) { + this.destroy(); throw new Error("maxBytes cannot be less than 1"); } if (this.#maxBackupCount < 1) { + this.destroy(); throw new Error("maxBackupCount cannot be less than 1"); } await super.setup(); @@ -170,29 +197,32 @@ export class RotatingFileHandler extends FileHandler { // Throw if any backups also exist for (let i = 1; i <= this.#maxBackupCount; i++) { if (await exists(this._filename + "." + i)) { - Deno.close(this._file.rid); + this.destroy(); throw new Deno.errors.AlreadyExists( "Backup log file " + this._filename + "." + i + " already exists" ); } } + } else { + this.#currentFileSize = (await stat(this._filename)).size; } } - handle(logRecord: LogRecord): void { - if (this.level > logRecord.level) return; + log(msg: string): void { + const msgByteLength = this._encoder.encode(msg).byteLength + 1; - const msg = this.format(logRecord); - const currentFileSize = statSync(this._filename).size; - if (currentFileSize + msg.length > this.#maxBytes) { + if (this.#currentFileSize + msgByteLength > this.#maxBytes) { this.rotateLogFiles(); + this.#currentFileSize = 0; } - return this.log(msg); + this._buf.writeSync(this._encoder.encode(msg + "\n")); + this.#currentFileSize += msgByteLength; } rotateLogFiles(): void { - close(this._file.rid); + this._buf.flush(); + close(this._file!.rid); for (let i = this.#maxBackupCount - 1; i >= 0; i--) { const source = this._filename + (i === 0 ? "" : "." + i); @@ -205,5 +235,6 @@ export class RotatingFileHandler extends FileHandler { this._file = openSync(this._filename, this._openOptions); this._writer = this._file; + this._buf = new BufWriterSync(this._file); } } diff --git a/std/log/handlers_test.ts b/std/log/handlers_test.ts index b69ef6eabd0c94..cb73fa56fa8965 100644 --- a/std/log/handlers_test.ts +++ b/std/log/handlers_test.ts @@ -1,6 +1,11 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. const { test } = Deno; -import { assert, assertEquals, assertThrowsAsync } from "../testing/asserts.ts"; +import { + assert, + assertEquals, + assertThrowsAsync, + assertNotEquals, +} from "../testing/asserts.ts"; import { LogLevels, LogLevelNames, @@ -115,18 +120,18 @@ test({ test({ name: "FileHandler with mode 'x' will throw if log file already exists", async fn() { - await assertThrowsAsync( - async () => { - Deno.writeFileSync(LOG_FILE, new TextEncoder().encode("hello world")); - const fileHandler = new FileHandler("WARNING", { - filename: LOG_FILE, - mode: "x", - }); - await fileHandler.setup(); - }, - Deno.errors.AlreadyExists, - "ile exists" - ); + const fileHandler = new FileHandler("WARNING", { + filename: LOG_FILE, + mode: "x", + }); + Deno.writeFileSync(LOG_FILE, new TextEncoder().encode("hello world")); + + await assertThrowsAsync(async () => { + await fileHandler.setup(); + }, Deno.errors.AlreadyExists); + + await fileHandler.destroy(); + Deno.removeSync(LOG_FILE); }, }); @@ -171,30 +176,32 @@ test({ name: "RotatingFileHandler with mode 'x' will throw if any log file already exists", async fn() { + Deno.writeFileSync( + LOG_FILE + ".3", + new TextEncoder().encode("hello world") + ); + const fileHandler = new RotatingFileHandler("WARNING", { + filename: LOG_FILE, + maxBytes: 50, + maxBackupCount: 3, + mode: "x", + }); await assertThrowsAsync( async () => { - Deno.writeFileSync( - LOG_FILE + ".3", - new TextEncoder().encode("hello world") - ); - const fileHandler = new RotatingFileHandler("WARNING", { - filename: LOG_FILE, - maxBytes: 50, - maxBackupCount: 3, - mode: "x", - }); await fileHandler.setup(); }, Deno.errors.AlreadyExists, "Backup log file " + LOG_FILE + ".3 already exists" ); + + fileHandler.destroy(); Deno.removeSync(LOG_FILE + ".3"); Deno.removeSync(LOG_FILE); }, }); test({ - name: "RotatingFileHandler with first rollover", + name: "RotatingFileHandler with first rollover, monitor step by step", async fn() { const fileHandler = new RotatingFileHandler("WARNING", { filename: LOG_FILE, @@ -205,16 +212,43 @@ test({ await fileHandler.setup(); fileHandler.handle(new LogRecord("AAA", [], LogLevels.ERROR)); // 'ERROR AAA\n' = 10 bytes + fileHandler.flush(); assertEquals((await Deno.stat(LOG_FILE)).size, 10); fileHandler.handle(new LogRecord("AAA", [], LogLevels.ERROR)); + fileHandler.flush(); assertEquals((await Deno.stat(LOG_FILE)).size, 20); fileHandler.handle(new LogRecord("AAA", [], LogLevels.ERROR)); + fileHandler.flush(); // Rollover occurred. Log file now has 1 record, rollover file has the original 2 assertEquals((await Deno.stat(LOG_FILE)).size, 10); assertEquals((await Deno.stat(LOG_FILE + ".1")).size, 20); + await fileHandler.destroy(); + + Deno.removeSync(LOG_FILE); + Deno.removeSync(LOG_FILE + ".1"); + }, +}); + +test({ + name: "RotatingFileHandler with first rollover, check all at once", + async fn() { + const fileHandler = new RotatingFileHandler("WARNING", { + filename: LOG_FILE, + maxBytes: 25, + maxBackupCount: 3, + mode: "w", + }); + await fileHandler.setup(); + + fileHandler.handle(new LogRecord("AAA", [], LogLevels.ERROR)); // 'ERROR AAA\n' = 10 bytes + fileHandler.handle(new LogRecord("AAA", [], LogLevels.ERROR)); + fileHandler.handle(new LogRecord("AAA", [], LogLevels.ERROR)); await fileHandler.destroy(); + assertEquals((await Deno.stat(LOG_FILE)).size, 10); + assertEquals((await Deno.stat(LOG_FILE + ".1")).size, 20); + Deno.removeSync(LOG_FILE); Deno.removeSync(LOG_FILE + ".1"); }, @@ -307,3 +341,82 @@ test({ ); }, }); + +test({ + name: "Window unload flushes buffer", + async fn() { + const fileHandler = new FileHandler("WARNING", { + filename: LOG_FILE, + mode: "w", + }); + await fileHandler.setup(); + fileHandler.handle(new LogRecord("AAA", [], LogLevels.ERROR)); // 'ERROR AAA\n' = 10 bytes + + assertEquals((await Deno.stat(LOG_FILE)).size, 0); + dispatchEvent(new Event("unload")); + assertEquals((await Deno.stat(LOG_FILE)).size, 10); + + Deno.removeSync(LOG_FILE); + }, +}); + +test({ + name: "RotatingFileHandler: rotate on byte length, not msg length", + async fn() { + const fileHandler = new RotatingFileHandler("WARNING", { + filename: LOG_FILE, + maxBytes: 7, + maxBackupCount: 1, + mode: "w", + }); + await fileHandler.setup(); + + const msg = "。"; + const msgLength = msg.length; + const msgByteLength = new TextEncoder().encode(msg).byteLength; + assertNotEquals(msgLength, msgByteLength); + assertEquals(msgLength, 1); + assertEquals(msgByteLength, 3); + + fileHandler.log(msg); // logs 4 bytes (including '\n') + fileHandler.log(msg); // max bytes is 7, but this would be 8. Rollover. + + await fileHandler.destroy(); + + const fileSize1 = (await Deno.stat(LOG_FILE)).size; + const fileSize2 = (await Deno.stat(LOG_FILE + ".1")).size; + + assertEquals(fileSize1, msgByteLength + 1); + assertEquals(fileSize2, msgByteLength + 1); + + Deno.removeSync(LOG_FILE); + Deno.removeSync(LOG_FILE + ".1"); + }, +}); + +test({ + name: "FileHandler: Critical logs trigger immediate flush", + async fn() { + const fileHandler = new FileHandler("WARNING", { + filename: LOG_FILE, + mode: "w", + }); + await fileHandler.setup(); + + fileHandler.handle(new LogRecord("AAA", [], LogLevels.ERROR)); + + // ERROR won't trigger immediate flush + const fileSize = (await Deno.stat(LOG_FILE)).size; + assertEquals(fileSize, 0); + + fileHandler.handle(new LogRecord("AAA", [], LogLevels.CRITICAL)); + + // CRITICAL will trigger immediate flush + const fileSize2 = (await Deno.stat(LOG_FILE)).size; + // ERROR record is 10 bytes, CRITICAL is 13 bytes + assertEquals(fileSize2, 23); + + await fileHandler.destroy(); + Deno.removeSync(LOG_FILE); + }, +}); diff --git a/std/log/mod.ts b/std/log/mod.ts index 983e82401542ec..e596c81ba8e58d 100644 --- a/std/log/mod.ts +++ b/std/log/mod.ts @@ -7,7 +7,7 @@ import { FileHandler, RotatingFileHandler, } from "./handlers.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; import { LevelName } from "./levels.ts"; export { LogLevels } from "./levels.ts"; diff --git a/std/mime/multipart.ts b/std/mime/multipart.ts index 75d418c92e3295..73a6544b5be2b9 100644 --- a/std/mime/multipart.ts +++ b/std/mime/multipart.ts @@ -12,7 +12,7 @@ import { extname } from "../path/mod.ts"; import { tempFile } from "../io/util.ts"; import { BufReader, BufWriter } from "../io/bufio.ts"; import { encoder } from "../encoding/utf8.ts"; -import { assertStrictEq, assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; import { TextProtoReader } from "../textproto/mod.ts"; import { hasOwnProperty } from "../_util/has_own_property.ts"; @@ -178,7 +178,7 @@ class PartReader implements Reader, Closer { ); if (this.n === 0) { // Force buffered I/O to read more into buffer. - assertStrictEq(eof, false); + assert(eof === false); peekLength++; } } @@ -190,7 +190,7 @@ class PartReader implements Reader, Closer { const nread = min(p.length, this.n); const buf = p.subarray(0, nread); const r = await br.readFull(buf); - assertStrictEq(r, buf); + assert(r === buf); this.n -= nread; this.total += nread; return nread; @@ -252,11 +252,13 @@ function skipLWSPChar(u: Uint8Array): Uint8Array { } export interface MultipartFormData { - file(key: string): FormFile | undefined; + file(key: string): FormFile | FormFile[] | undefined; value(key: string): string | undefined; - entries(): IterableIterator<[string, string | FormFile | undefined]>; + entries(): IterableIterator< + [string, string | FormFile | FormFile[] | undefined] + >; [Symbol.iterator](): IterableIterator< - [string, string | FormFile | undefined] + [string, string | FormFile | FormFile[] | undefined] >; /** Remove all tempfiles */ removeAll(): Promise; @@ -282,7 +284,7 @@ export class MultipartReader { * @param maxMemory maximum memory size to store file in memory. bytes. @default 10485760 (10MB) * */ async readForm(maxMemory = 10 << 20): Promise { - const fileMap = new Map(); + const fileMap = new Map(); const valueMap = new Map(); let maxValueBytes = maxMemory + (10 << 20); const buf = new Buffer(new Uint8Array(maxValueBytes)); @@ -307,7 +309,7 @@ export class MultipartReader { continue; } // file - let formFile: FormFile | undefined; + let formFile: FormFile | FormFile[] | undefined; const n = await copyN(p, buf, maxValueBytes); const contentType = p.headers.get("content-type"); assert(contentType != null, "content-type must be set"); @@ -343,7 +345,16 @@ export class MultipartReader { maxValueBytes -= n; } if (formFile) { - fileMap.set(p.formName, formFile); + const mapVal = fileMap.get(p.formName); + if (mapVal !== undefined) { + if (Array.isArray(mapVal)) { + mapVal.push(formFile); + } else { + fileMap.set(p.formName, [mapVal, formFile]); + } + } else { + fileMap.set(p.formName, formFile); + } } } return multipatFormData(fileMap, valueMap); @@ -411,17 +422,17 @@ export class MultipartReader { } function multipatFormData( - fileMap: Map, + fileMap: Map, valueMap: Map ): MultipartFormData { - function file(key: string): FormFile | undefined { + function file(key: string): FormFile | FormFile[] | undefined { return fileMap.get(key); } function value(key: string): string | undefined { return valueMap.get(key); } function* entries(): IterableIterator< - [string, string | FormFile | undefined] + [string, string | FormFile | FormFile[] | undefined] > { yield* fileMap; yield* valueMap; @@ -429,8 +440,15 @@ function multipatFormData( async function removeAll(): Promise { const promises: Array> = []; for (const val of fileMap.values()) { - if (!val.tempfile) continue; - promises.push(Deno.remove(val.tempfile)); + if (Array.isArray(val)) { + for (const subVal of val) { + if (!subVal.tempfile) continue; + promises.push(Deno.remove(subVal.tempfile)); + } + } else { + if (!val.tempfile) continue; + promises.push(Deno.remove(val.tempfile)); + } } await Promise.all(promises); } @@ -440,7 +458,7 @@ function multipatFormData( entries, removeAll, [Symbol.iterator](): IterableIterator< - [string, string | FormFile | undefined] + [string, string | FormFile | FormFile[] | undefined] > { return entries(); }, diff --git a/std/mime/multipart_test.ts b/std/mime/multipart_test.ts index 61afdba03635bf..858dc3919a2e7a 100644 --- a/std/mime/multipart_test.ts +++ b/std/mime/multipart_test.ts @@ -145,8 +145,8 @@ test("multipartMultipartWriter3", async function (): Promise { ); await assertThrowsAsync( async (): Promise => { - // @ts-ignore - await mw.writeFile("bar", "file", null); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await mw.writeFile("bar", "file", null as any); }, Error, "closed" @@ -225,7 +225,10 @@ test({ try { assertEquals(form.value("deno"), "land"); assertEquals(form.value("bar"), "bar"); - const file = form.file("file"); + let file = form.file("file"); + if (Array.isArray(file)) { + file = file[0]; + } assert(file != null); assert(file.tempfile != null); assertEquals(file.size, size); @@ -249,7 +252,10 @@ test({ "--------------------------434049563556637648550474" ); const form = await mr.readForm(20); - const file = form.file("file"); + let file = form.file("file"); + if (Array.isArray(file)) { + file = file[0]; + } assert(file != null); const { tempfile, content } = file; assert(tempfile != null); diff --git a/std/node/_fs/_fs_common.ts b/std/node/_fs/_fs_common.ts index 1f00bc4814c5de..bf7c0f67558afa 100644 --- a/std/node/_fs/_fs_common.ts +++ b/std/node/_fs/_fs_common.ts @@ -35,6 +35,11 @@ export function getEncoding( const encoding = typeof optOrCallback === "string" ? optOrCallback : optOrCallback.encoding; if (!encoding) return null; + return encoding; +} + +export function checkEncoding(encoding: string | null): string | null { + if (!encoding) return null; if (encoding === "utf8" || encoding === "utf-8") { return "utf8"; } diff --git a/std/node/_fs/_fs_copy_test.ts b/std/node/_fs/_fs_copy_test.ts index 9a27f7a83fe66a..f7ce0e279ec5b6 100644 --- a/std/node/_fs/_fs_copy_test.ts +++ b/std/node/_fs/_fs_copy_test.ts @@ -10,13 +10,13 @@ const destFile = "./destination.txt"; test({ name: "[std/node/fs] copy file", fn: async () => { - const srouceFile = Deno.makeTempFileSync(); + const sourceFile = Deno.makeTempFileSync(); const err = await new Promise((resolve) => { - copyFile(srouceFile, destFile, (err?: Error | null) => resolve(err)); + copyFile(sourceFile, destFile, (err?: Error | null) => resolve(err)); }); assert(!err); assert(existsSync(destFile)); - Deno.removeSync(srouceFile); + Deno.removeSync(sourceFile); Deno.removeSync(destFile); }, }); @@ -24,10 +24,10 @@ test({ test({ name: "[std/node/fs] copy file sync", fn: () => { - const srouceFile = Deno.makeTempFileSync(); - copyFileSync(srouceFile, destFile); + const sourceFile = Deno.makeTempFileSync(); + copyFileSync(sourceFile, destFile); assert(existsSync(destFile)); - Deno.removeSync(srouceFile); + Deno.removeSync(sourceFile); Deno.removeSync(destFile); }, }); diff --git a/std/node/_fs/_fs_dir.ts b/std/node/_fs/_fs_dir.ts index a39422ce06d873..7f2085b3b1b97f 100644 --- a/std/node/_fs/_fs_dir.ts +++ b/std/node/_fs/_fs_dir.ts @@ -1,5 +1,5 @@ import Dirent from "./_fs_dirent.ts"; -import { assert } from "../../testing/asserts.ts"; +import { assert } from "../../_util/assert.ts"; export default class Dir { private dirPath: string | Uint8Array; diff --git a/std/node/_fs/_fs_link.ts b/std/node/_fs/_fs_link.ts new file mode 100644 index 00000000000000..50916a7ba0d7ef --- /dev/null +++ b/std/node/_fs/_fs_link.ts @@ -0,0 +1,37 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +import { CallbackWithError } from "./_fs_common.ts"; +import { fromFileUrl } from "../path.ts"; + +/** + * TODO: Also accept 'path' parameter as a Node polyfill Buffer type once these + * are implemented. See https://github.com/denoland/deno/issues/3403 + */ +export function link( + existingPath: string | URL, + newPath: string | URL, + callback: CallbackWithError +): void { + existingPath = + existingPath instanceof URL ? fromFileUrl(existingPath) : existingPath; + newPath = newPath instanceof URL ? fromFileUrl(newPath) : newPath; + + Deno.link(existingPath, newPath) + .then(() => callback()) + .catch(callback); +} + +/** + * TODO: Also accept 'path' parameter as a Node polyfill Buffer type once these + * are implemented. See https://github.com/denoland/deno/issues/3403 + */ +export function linkSync( + existingPath: string | URL, + newPath: string | URL +): void { + existingPath = + existingPath instanceof URL ? fromFileUrl(existingPath) : existingPath; + newPath = newPath instanceof URL ? fromFileUrl(newPath) : newPath; + + Deno.linkSync(existingPath, newPath); +} diff --git a/std/node/_fs/_fs_link_test.ts b/std/node/_fs/_fs_link_test.ts new file mode 100644 index 00000000000000..e59984c8cbcfb0 --- /dev/null +++ b/std/node/_fs/_fs_link_test.ts @@ -0,0 +1,67 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +const { test } = Deno; +import { fail, assertEquals } from "../../testing/asserts.ts"; +import { link, linkSync } from "./_fs_link.ts"; +import { assert } from "https://deno.land/std@v0.50.0/testing/asserts.ts"; +const isWindows = Deno.build.os === "windows"; + +test({ + ignore: isWindows, + name: "ASYNC: hard linking files works as expected", + async fn() { + const tempFile: string = await Deno.makeTempFile(); + const linkedFile: string = tempFile + ".link"; + await new Promise((res, rej) => { + link(tempFile, linkedFile, (err) => { + if (err) rej(err); + else res(); + }); + }) + .then(() => { + assertEquals(Deno.statSync(tempFile), Deno.statSync(linkedFile)); + }) + .catch(() => { + fail("Expected to succeed"); + }) + .finally(() => { + Deno.removeSync(tempFile); + Deno.removeSync(linkedFile); + }); + }, +}); + +test({ + ignore: isWindows, + name: "ASYNC: hard linking files passes error to callback", + async fn() { + let failed = false; + await new Promise((res, rej) => { + link("no-such-file", "no-such-file", (err) => { + if (err) rej(err); + else res(); + }); + }) + .then(() => { + fail("Expected to succeed"); + }) + .catch((err) => { + assert(err); + failed = true; + }); + assert(failed); + }, +}); + +test({ + ignore: isWindows, + name: "SYNC: hard linking files works as expected", + fn() { + const tempFile: string = Deno.makeTempFileSync(); + const linkedFile: string = tempFile + ".link"; + linkSync(tempFile, linkedFile); + + assertEquals(Deno.statSync(tempFile), Deno.statSync(linkedFile)); + Deno.removeSync(tempFile); + Deno.removeSync(linkedFile); + }, +}); diff --git a/std/node/_fs/_fs_readFile.ts b/std/node/_fs/_fs_readFile.ts index ccea71cd67210c..448045fd24a3ca 100644 --- a/std/node/_fs/_fs_readFile.ts +++ b/std/node/_fs/_fs_readFile.ts @@ -3,23 +3,23 @@ import { intoCallbackAPIWithIntercept, MaybeEmpty } from "../_utils.ts"; import { getEncoding, FileOptions } from "./_fs_common.ts"; +import { Buffer } from "../buffer.ts"; import { fromFileUrl } from "../path.ts"; const { readFile: denoReadFile, readFileSync: denoReadFileSync } = Deno; type ReadFileCallback = ( err: MaybeEmpty, - data: MaybeEmpty + data: MaybeEmpty ) => void; function maybeDecode( data: Uint8Array, encoding: string | null -): string | Uint8Array { - if (encoding === "utf8") { - return new TextDecoder().decode(data); - } - return data; +): string | Buffer { + const buffer = new Buffer(data.buffer, data.byteOffset, data.byteLength); + if (encoding) return buffer.toString(encoding); + return buffer; } export function readFile( @@ -37,9 +37,9 @@ export function readFile( const encoding = getEncoding(optOrCallback); - intoCallbackAPIWithIntercept( + intoCallbackAPIWithIntercept( denoReadFile, - (data: Uint8Array): string | Uint8Array => maybeDecode(data, encoding), + (data: Uint8Array): string | Buffer => maybeDecode(data, encoding), cb, path ); @@ -48,7 +48,7 @@ export function readFile( export function readFileSync( path: string | URL, opt?: FileOptions | string -): string | Uint8Array { +): string | Buffer { path = path instanceof URL ? fromFileUrl(path) : path; return maybeDecode(denoReadFileSync(path), getEncoding(opt)); } diff --git a/std/node/_fs/_fs_writeFile.ts b/std/node/_fs/_fs_writeFile.ts index 72e75c97a6f881..4ede4263832960 100644 --- a/std/node/_fs/_fs_writeFile.ts +++ b/std/node/_fs/_fs_writeFile.ts @@ -7,6 +7,7 @@ import { CallbackWithError, isFileOptions, getEncoding, + checkEncoding, getOpenOptions, } from "./_fs_common.ts"; @@ -35,7 +36,7 @@ export function writeFile( ? options.mode : undefined; - const encoding = getEncoding(options) || "utf8"; + const encoding = checkEncoding(getEncoding(options)) || "utf8"; const openOptions = getOpenOptions(flag || "w"); if (typeof data === "string" && encoding === "utf8") @@ -82,7 +83,7 @@ export function writeFileSync( ? options.mode : undefined; - const encoding = getEncoding(options) || "utf8"; + const encoding = checkEncoding(getEncoding(options)) || "utf8"; const openOptions = getOpenOptions(flag || "w"); if (typeof data === "string" && encoding === "utf8") diff --git a/std/node/_util/_util_callbackify.ts b/std/node/_util/_util_callbackify.ts index fdcaed526823a1..4c752e7ef729b2 100644 --- a/std/node/_util/_util_callbackify.ts +++ b/std/node/_util/_util_callbackify.ts @@ -100,7 +100,7 @@ function callbackify(original: any): any { }; original.apply(this, args).then( (ret: unknown) => { - setTimeout(cb.bind(this, null, ret), 0); + queueMicrotask(cb.bind(this, null, ret)); }, (rej: unknown) => { rej = rej || new NodeFalsyValueRejectionError(rej); diff --git a/std/node/_util/_util_callbackify_test.ts b/std/node/_util/_util_callbackify_test.ts index b4dcae75507ad3..c68a2ed50c03e4 100644 --- a/std/node/_util/_util_callbackify_test.ts +++ b/std/node/_util/_util_callbackify_test.ts @@ -24,7 +24,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ const { test } = Deno; -import { assert, assertStrictEq } from "../../testing/asserts.ts"; +import { assert, assertStrictEquals } from "../../testing/asserts.ts"; import { callbackify } from "./_util_callbackify.ts"; const values = [ @@ -89,8 +89,8 @@ test("callbackify passes the resolution value as the second argument to the call const cbAsyncFn = callbackify(asyncFn); testQueue.enqueue((done) => { cbAsyncFn((err: unknown, ret: unknown) => { - assertStrictEq(err, null); - assertStrictEq(ret, value); + assertStrictEquals(err, null); + assertStrictEquals(ret, value); done(); }); }); @@ -101,8 +101,8 @@ test("callbackify passes the resolution value as the second argument to the call const cbPromiseFn = callbackify(promiseFn); testQueue.enqueue((done) => { cbPromiseFn((err: unknown, ret: unknown) => { - assertStrictEq(err, null); - assertStrictEq(ret, value); + assertStrictEquals(err, null); + assertStrictEquals(ret, value); done(); }); }); @@ -119,8 +119,8 @@ test("callbackify passes the resolution value as the second argument to the call const cbThenableFn = callbackify(thenableFn); testQueue.enqueue((done) => { cbThenableFn((err: unknown, ret: unknown) => { - assertStrictEq(err, null); - assertStrictEq(ret, value); + assertStrictEquals(err, null); + assertStrictEquals(ret, value); done(); }); }); @@ -138,21 +138,21 @@ test("callbackify passes the rejection value as the first argument to the callba return Promise.reject(value); } const cbAsyncFn = callbackify(asyncFn); - assertStrictEq(cbAsyncFn.length, 1); - assertStrictEq(cbAsyncFn.name, "asyncFnCallbackified"); + assertStrictEquals(cbAsyncFn.length, 1); + assertStrictEquals(cbAsyncFn.name, "asyncFnCallbackified"); testQueue.enqueue((done) => { cbAsyncFn((err: unknown, ret: unknown) => { - assertStrictEq(ret, undefined); + assertStrictEquals(ret, undefined); if (err instanceof Error) { if ("reason" in err) { assert(!value); - assertStrictEq((err as any).code, "ERR_FALSY_VALUE_REJECTION"); - assertStrictEq((err as any).reason, value); + assertStrictEquals((err as any).code, "ERR_FALSY_VALUE_REJECTION"); + assertStrictEquals((err as any).reason, value); } else { - assertStrictEq(String(value).endsWith(err.message), true); + assertStrictEquals(String(value).endsWith(err.message), true); } } else { - assertStrictEq(err, value); + assertStrictEquals(err, value); } done(); }); @@ -170,20 +170,20 @@ test("callbackify passes the rejection value as the first argument to the callba }); const cbPromiseFn = callbackify(promiseFn); - assertStrictEq(promiseFn.name, obj); + assertStrictEquals(promiseFn.name, obj); testQueue.enqueue((done) => { cbPromiseFn((err: unknown, ret: unknown) => { - assertStrictEq(ret, undefined); + assertStrictEquals(ret, undefined); if (err instanceof Error) { if ("reason" in err) { assert(!value); - assertStrictEq((err as any).code, "ERR_FALSY_VALUE_REJECTION"); - assertStrictEq((err as any).reason, value); + assertStrictEquals((err as any).code, "ERR_FALSY_VALUE_REJECTION"); + assertStrictEquals((err as any).reason, value); } else { - assertStrictEq(String(value).endsWith(err.message), true); + assertStrictEquals(String(value).endsWith(err.message), true); } } else { - assertStrictEq(err, value); + assertStrictEquals(err, value); } done(); }); @@ -202,17 +202,17 @@ test("callbackify passes the rejection value as the first argument to the callba const cbThenableFn = callbackify(thenableFn); testQueue.enqueue((done) => { cbThenableFn((err: unknown, ret: unknown) => { - assertStrictEq(ret, undefined); + assertStrictEquals(ret, undefined); if (err instanceof Error) { if ("reason" in err) { assert(!value); - assertStrictEq((err as any).code, "ERR_FALSY_VALUE_REJECTION"); - assertStrictEq((err as any).reason, value); + assertStrictEquals((err as any).code, "ERR_FALSY_VALUE_REJECTION"); + assertStrictEquals((err as any).reason, value); } else { - assertStrictEq(String(value).endsWith(err.message), true); + assertStrictEquals(String(value).endsWith(err.message), true); } } else { - assertStrictEq(err, value); + assertStrictEquals(err, value); } done(); }); @@ -228,24 +228,24 @@ test("callbackify passes arguments to the original", async () => { for (const value of values) { // eslint-disable-next-line require-await async function asyncFn(arg: T): Promise { - assertStrictEq(arg, value); + assertStrictEquals(arg, value); return arg; } const cbAsyncFn = callbackify(asyncFn); - assertStrictEq(cbAsyncFn.length, 2); + assertStrictEquals(cbAsyncFn.length, 2); assert(Object.getPrototypeOf(cbAsyncFn) !== Object.getPrototypeOf(asyncFn)); - assertStrictEq(Object.getPrototypeOf(cbAsyncFn), Function.prototype); + assertStrictEquals(Object.getPrototypeOf(cbAsyncFn), Function.prototype); testQueue.enqueue((done) => { cbAsyncFn(value, (err: unknown, ret: unknown) => { - assertStrictEq(err, null); - assertStrictEq(ret, value); + assertStrictEquals(err, null); + assertStrictEquals(ret, value); done(); }); }); function promiseFn(arg: T): Promise { - assertStrictEq(arg, value); + assertStrictEquals(arg, value); return Promise.resolve(arg); } const obj = {}; @@ -257,11 +257,11 @@ test("callbackify passes arguments to the original", async () => { }); const cbPromiseFn = callbackify(promiseFn); - assertStrictEq(promiseFn.length, obj); + assertStrictEquals(promiseFn.length, obj); testQueue.enqueue((done) => { cbPromiseFn(value, (err: unknown, ret: unknown) => { - assertStrictEq(err, null); - assertStrictEq(ret, value); + assertStrictEquals(err, null); + assertStrictEquals(ret, value); done(); }); }); @@ -276,7 +276,7 @@ test("callbackify preserves the `this` binding", async () => { for (const value of values) { const objectWithSyncFunction = { fn(this: unknown, arg: typeof value): Promise { - assertStrictEq(this, objectWithSyncFunction); + assertStrictEquals(this, objectWithSyncFunction); return Promise.resolve(arg); }, }; @@ -287,9 +287,9 @@ test("callbackify preserves the `this` binding", async () => { err: unknown, ret: unknown ) { - assertStrictEq(err, null); - assertStrictEq(ret, value); - assertStrictEq(this, objectWithSyncFunction); + assertStrictEquals(err, null); + assertStrictEquals(ret, value); + assertStrictEquals(this, objectWithSyncFunction); done(); }); }); @@ -297,7 +297,7 @@ test("callbackify preserves the `this` binding", async () => { const objectWithAsyncFunction = { // eslint-disable-next-line require-await async fn(this: unknown, arg: typeof value): Promise { - assertStrictEq(this, objectWithAsyncFunction); + assertStrictEquals(this, objectWithAsyncFunction); return arg; }, }; @@ -308,9 +308,9 @@ test("callbackify preserves the `this` binding", async () => { err: unknown, ret: unknown ) { - assertStrictEq(err, null); - assertStrictEq(ret, value); - assertStrictEq(this, objectWithAsyncFunction); + assertStrictEquals(err, null); + assertStrictEquals(ret, value); + assertStrictEquals(this, objectWithAsyncFunction); done(); }); }); @@ -326,9 +326,9 @@ test("callbackify throws with non-function inputs", () => { throw Error("We should never reach this error"); } catch (err) { assert(err instanceof TypeError); - assertStrictEq((err as any).code, "ERR_INVALID_ARG_TYPE"); - assertStrictEq(err.name, "TypeError"); - assertStrictEq( + assertStrictEquals((err as any).code, "ERR_INVALID_ARG_TYPE"); + assertStrictEquals(err.name, "TypeError"); + assertStrictEquals( err.message, 'The "original" argument must be of type function.' ); @@ -353,9 +353,9 @@ test("callbackify returns a function that throws if the last argument is not a f throw Error("We should never reach this error"); } catch (err) { assert(err instanceof TypeError); - assertStrictEq((err as any).code, "ERR_INVALID_ARG_TYPE"); - assertStrictEq(err.name, "TypeError"); - assertStrictEq( + assertStrictEquals((err as any).code, "ERR_INVALID_ARG_TYPE"); + assertStrictEquals(err.name, "TypeError"); + assertStrictEquals( err.message, "The last argument must be of type function." ); diff --git a/std/node/_util/_util_types.ts b/std/node/_util/_util_types.ts new file mode 100644 index 00000000000000..df8ccbb055e744 --- /dev/null +++ b/std/node/_util/_util_types.ts @@ -0,0 +1,232 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +// +// Adapted from Node.js. Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +const _toString = Object.prototype.toString; + +const _isObjectLike = (value: unknown): boolean => + value !== null && typeof value === "object"; + +const _isFunctionLike = (value: unknown): boolean => + value !== null && typeof value === "function"; + +export function isAnyArrayBuffer(value: unknown): boolean { + return ( + _isObjectLike(value) && + (_toString.call(value) === "[object ArrayBuffer]" || + _toString.call(value) === "[object SharedArrayBuffer]") + ); +} + +export function isArrayBufferView(value: unknown): boolean { + return ArrayBuffer.isView(value); +} + +export function isArgumentsObject(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Arguments]"; +} + +export function isArrayBuffer(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object ArrayBuffer]" + ); +} + +export function isAsyncFunction(value: unknown): boolean { + return ( + _isFunctionLike(value) && _toString.call(value) === "[object AsyncFunction]" + ); +} + +export function isBigInt64Array(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object BigInt64Array]" + ); +} + +export function isBigUint64Array(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object BigUint64Array]" + ); +} + +export function isBooleanObject(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Boolean]"; +} + +export function isBoxedPrimitive(value: unknown): boolean { + return ( + isBooleanObject(value) || + isStringObject(value) || + isNumberObject(value) || + isSymbolObject(value) || + isBigIntObject(value) + ); +} + +export function isDataView(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object DataView]"; +} + +export function isDate(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Date]"; +} + +// isExternal: Not implemented + +export function isFloat32Array(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object Float32Array]" + ); +} + +export function isFloat64Array(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object Float64Array]" + ); +} + +export function isGeneratorFunction(value: unknown): boolean { + return ( + _isFunctionLike(value) && + _toString.call(value) === "[object GeneratorFunction]" + ); +} + +export function isGeneratorObject(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Generator]"; +} + +export function isInt8Array(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Int8Array]"; +} + +export function isInt16Array(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object Int16Array]" + ); +} + +export function isInt32Array(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object Int32Array]" + ); +} + +export function isMap(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Map]"; +} + +export function isMapIterator(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object Map Iterator]" + ); +} + +export function isModuleNamespaceObject(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Module]"; +} + +export function isNativeError(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Error]"; +} + +export function isNumberObject(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Number]"; +} + +export function isBigIntObject(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object BigInt]"; +} + +export function isPromise(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Promise]"; +} + +export function isRegExp(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object RegExp]"; +} + +export function isSet(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Set]"; +} + +export function isSetIterator(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object Set Iterator]" + ); +} + +export function isSharedArrayBuffer(value: unknown): boolean { + return ( + _isObjectLike(value) && + _toString.call(value) === "[object SharedArrayBuffer]" + ); +} + +export function isStringObject(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object String]"; +} + +export function isSymbolObject(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object Symbol]"; +} + +// Adapted from Lodash +export function isTypedArray(value: unknown): boolean { + /** Used to match `toStringTag` values of typed arrays. */ + const reTypedTag = /^\[object (?:Float(?:32|64)|(?:Int|Uint)(?:8|16|32)|Uint8Clamped)Array\]$/; + return _isObjectLike(value) && reTypedTag.test(_toString.call(value)); +} + +export function isUint8Array(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object Uint8Array]" + ); +} + +export function isUint8ClampedArray(value: unknown): boolean { + return ( + _isObjectLike(value) && + _toString.call(value) === "[object Uint8ClampedArray]" + ); +} + +export function isUint16Array(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object Uint16Array]" + ); +} + +export function isUint32Array(value: unknown): boolean { + return ( + _isObjectLike(value) && _toString.call(value) === "[object Uint32Array]" + ); +} + +export function isWeakMap(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object WeakMap]"; +} + +export function isWeakSet(value: unknown): boolean { + return _isObjectLike(value) && _toString.call(value) === "[object WeakSet]"; +} diff --git a/std/node/_util/_util_types_test.ts b/std/node/_util/_util_types_test.ts new file mode 100644 index 00000000000000..2d4307e7755a35 --- /dev/null +++ b/std/node/_util/_util_types_test.ts @@ -0,0 +1,511 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +// +// Adapted from Node.js. Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +const { test } = Deno; +import { assertStrictEquals } from "../../testing/asserts.ts"; +import { + isDate, + isMap, + isSet, + isAnyArrayBuffer, + isArrayBufferView, + isArgumentsObject, + isArrayBuffer, + isStringObject, + isAsyncFunction, + isBigInt64Array, + isBigUint64Array, + isBooleanObject, + isBoxedPrimitive, + isDataView, + isFloat32Array, + isFloat64Array, + isGeneratorFunction, + isGeneratorObject, + isInt8Array, + isInt16Array, + isInt32Array, + isMapIterator, + isModuleNamespaceObject, + isNativeError, + isSymbolObject, + isTypedArray, + isUint8Array, + isUint8ClampedArray, + isUint16Array, + isUint32Array, + isNumberObject, + isBigIntObject, + isPromise, + isRegExp, + isSetIterator, + isSharedArrayBuffer, + isWeakMap, + isWeakSet, +} from "./_util_types.ts"; + +// Used to test isModuleNamespaceObject +import * as testModuleNamespaceOpbject from "./_util_types.ts"; + +// isAnyArrayBuffer +test("Should return true for valid ArrayBuffer types", () => { + assertStrictEquals(isAnyArrayBuffer(new ArrayBuffer(0)), true); + assertStrictEquals(isAnyArrayBuffer(new SharedArrayBuffer(0)), true); +}); + +test("Should return false for invalid ArrayBuffer types", () => { + assertStrictEquals(isAnyArrayBuffer({}), false); + assertStrictEquals(isAnyArrayBuffer([]), false); + assertStrictEquals(isAnyArrayBuffer(new Error()), false); +}); + +// isArrayBufferView +test("Should return true for valid ArrayBufferView types", () => { + assertStrictEquals(isArrayBufferView(new Int8Array(0)), true); + assertStrictEquals(isArrayBufferView(new Uint8Array(0)), true); + assertStrictEquals(isArrayBufferView(new Uint8ClampedArray(0)), true); + assertStrictEquals(isArrayBufferView(new Int16Array(0)), true); + assertStrictEquals(isArrayBufferView(new Uint16Array(0)), true); + assertStrictEquals(isArrayBufferView(new Int32Array(0)), true); + assertStrictEquals(isArrayBufferView(new Uint32Array(0)), true); + assertStrictEquals(isArrayBufferView(new Float32Array(0)), true); + assertStrictEquals(isArrayBufferView(new Float64Array(0)), true); + assertStrictEquals(isArrayBufferView(new DataView(new ArrayBuffer(0))), true); +}); + +test("Should return false for invalid ArrayBufferView types", () => { + assertStrictEquals(isArrayBufferView({}), false); + assertStrictEquals(isArrayBufferView([]), false); + assertStrictEquals(isArrayBufferView(new Error()), false); + assertStrictEquals(isArrayBufferView(new ArrayBuffer(0)), false); +}); + +// isArgumentsObject +// Note: not testable in TS + +test("Should return false for invalid Argument types", () => { + assertStrictEquals(isArgumentsObject({}), false); + assertStrictEquals(isArgumentsObject([]), false); + assertStrictEquals(isArgumentsObject(new Error()), false); +}); + +// isArrayBuffer +test("Should return true for valid ArrayBuffer types", () => { + assertStrictEquals(isArrayBuffer(new ArrayBuffer(0)), true); +}); + +test("Should return false for invalid ArrayBuffer types", () => { + assertStrictEquals(isArrayBuffer(new SharedArrayBuffer(0)), false); + assertStrictEquals(isArrayBuffer({}), false); + assertStrictEquals(isArrayBuffer([]), false); + assertStrictEquals(isArrayBuffer(new Error()), false); +}); + +// isAsyncFunction +test("Should return true for valid async function types", () => { + const asyncFunction = async (): Promise => {}; + assertStrictEquals(isAsyncFunction(asyncFunction), true); +}); + +test("Should return false for invalid async function types", () => { + const syncFunction = (): void => {}; + assertStrictEquals(isAsyncFunction(syncFunction), false); + assertStrictEquals(isAsyncFunction({}), false); + assertStrictEquals(isAsyncFunction([]), false); + assertStrictEquals(isAsyncFunction(new Error()), false); +}); + +// isBigInt64Array +test("Should return true for valid BigInt64Array types", () => { + assertStrictEquals(isBigInt64Array(new BigInt64Array()), true); +}); + +test("Should return false for invalid BigInt64Array types", () => { + assertStrictEquals(isBigInt64Array(new BigUint64Array()), false); + assertStrictEquals(isBigInt64Array(new Float32Array()), false); + assertStrictEquals(isBigInt64Array(new Int32Array()), false); +}); + +// isBigUint64Array +test("Should return true for valid isBigUint64Array types", () => { + assertStrictEquals(isBigUint64Array(new BigUint64Array()), true); +}); + +test("Should return false for invalid isBigUint64Array types", () => { + assertStrictEquals(isBigUint64Array(new BigInt64Array()), false); + assertStrictEquals(isBigUint64Array(new Float32Array()), false); + assertStrictEquals(isBigUint64Array(new Int32Array()), false); +}); + +// isBooleanObject +test("Should return true for valid Boolean object types", () => { + assertStrictEquals(isBooleanObject(new Boolean(false)), true); + assertStrictEquals(isBooleanObject(new Boolean(true)), true); +}); + +test("Should return false for invalid isBigUint64Array types", () => { + assertStrictEquals(isBooleanObject(false), false); + assertStrictEquals(isBooleanObject(true), false); + assertStrictEquals(isBooleanObject(Boolean(false)), false); + assertStrictEquals(isBooleanObject(Boolean(true)), false); +}); + +// isBoxedPrimitive +test("Should return true for valid boxed primitive values", () => { + assertStrictEquals(isBoxedPrimitive(new Boolean(false)), true); + assertStrictEquals(isBoxedPrimitive(Object(Symbol("foo"))), true); + assertStrictEquals(isBoxedPrimitive(Object(BigInt(5))), true); + assertStrictEquals(isBoxedPrimitive(new String("foo")), true); +}); + +test("Should return false for invalid boxed primitive values", () => { + assertStrictEquals(isBoxedPrimitive(false), false); + assertStrictEquals(isBoxedPrimitive(Symbol("foo")), false); +}); + +// isDateView +test("Should return true for valid DataView types", () => { + assertStrictEquals(isDataView(new DataView(new ArrayBuffer(0))), true); +}); + +test("Should return false for invalid DataView types", () => { + assertStrictEquals(isDataView(new Float64Array(0)), false); +}); + +// isDate +test("Should return true for valid date types", () => { + assertStrictEquals(isDate(new Date()), true); + assertStrictEquals(isDate(new Date(0)), true); + assertStrictEquals(isDate(new (eval("Date"))()), true); +}); + +test("Should return false for invalid date types", () => { + assertStrictEquals(isDate(Date()), false); + assertStrictEquals(isDate({}), false); + assertStrictEquals(isDate([]), false); + assertStrictEquals(isDate(new Error()), false); + assertStrictEquals(isDate(Object.create(Date.prototype)), false); +}); + +// isFloat32Array +test("Should return true for valid Float32Array types", () => { + assertStrictEquals(isFloat32Array(new Float32Array(0)), true); +}); + +test("Should return false for invalid Float32Array types", () => { + assertStrictEquals(isFloat32Array(new ArrayBuffer(0)), false); + assertStrictEquals(isFloat32Array(new Float64Array(0)), false); +}); + +// isFloat64Array +test("Should return true for valid Float64Array types", () => { + assertStrictEquals(isFloat64Array(new Float64Array(0)), true); +}); + +test("Should return false for invalid Float64Array types", () => { + assertStrictEquals(isFloat64Array(new ArrayBuffer(0)), false); + assertStrictEquals(isFloat64Array(new Uint8Array(0)), false); +}); + +// isGeneratorFunction +test("Should return true for valid generator functions", () => { + assertStrictEquals( + isGeneratorFunction(function* foo() {}), + true + ); +}); + +test("Should return false for invalid generator functions", () => { + assertStrictEquals( + isGeneratorFunction(function foo() {}), + false + ); +}); + +// isGeneratorObject +test("Should return true for valid generator object types", () => { + function* foo(): Iterator {} + assertStrictEquals(isGeneratorObject(foo()), true); +}); + +test("Should return false for invalid generation object types", () => { + assertStrictEquals( + isGeneratorObject(function* foo() {}), + false + ); +}); + +// isInt8Array +test("Should return true for valid Int8Array types", () => { + assertStrictEquals(isInt8Array(new Int8Array(0)), true); +}); + +test("Should return false for invalid Int8Array types", () => { + assertStrictEquals(isInt8Array(new ArrayBuffer(0)), false); + assertStrictEquals(isInt8Array(new Float64Array(0)), false); +}); + +// isInt16Array +test("Should return true for valid Int16Array types", () => { + assertStrictEquals(isInt16Array(new Int16Array(0)), true); +}); + +test("Should return false for invalid Int16Array type", () => { + assertStrictEquals(isInt16Array(new ArrayBuffer(0)), false); + assertStrictEquals(isInt16Array(new Float64Array(0)), false); +}); + +// isInt32Array +test("Should return true for valid isInt32Array types", () => { + assertStrictEquals(isInt32Array(new Int32Array(0)), true); +}); + +test("Should return false for invalid isInt32Array type", () => { + assertStrictEquals(isInt32Array(new ArrayBuffer(0)), false); + assertStrictEquals(isInt32Array(new Float64Array(0)), false); +}); + +// isStringObject +test("Should return true for valid String types", () => { + assertStrictEquals(isStringObject(new String("")), true); + assertStrictEquals(isStringObject(new String("Foo")), true); +}); + +test("Should return false for invalid String types", () => { + assertStrictEquals(isStringObject(""), false); + assertStrictEquals(isStringObject("Foo"), false); +}); + +// isMap +test("Should return true for valid Map types", () => { + assertStrictEquals(isMap(new Map()), true); +}); + +test("Should return false for invalid Map types", () => { + assertStrictEquals(isMap({}), false); + assertStrictEquals(isMap([]), false); + assertStrictEquals(isMap(new Date()), false); + assertStrictEquals(isMap(new Error()), false); +}); + +// isMapIterator +test("Should return true for valid Map Iterator types", () => { + const map = new Map(); + assertStrictEquals(isMapIterator(map.keys()), true); + assertStrictEquals(isMapIterator(map.values()), true); + assertStrictEquals(isMapIterator(map.entries()), true); + assertStrictEquals(isMapIterator(map[Symbol.iterator]()), true); +}); + +test("Should return false for invalid Map iterator types", () => { + assertStrictEquals(isMapIterator(new Map()), false); + assertStrictEquals(isMapIterator([]), false); + assertStrictEquals(isMapIterator(new Date()), false); + assertStrictEquals(isMapIterator(new Error()), false); +}); + +// isModuleNamespaceObject +test("Should return true for valid module namespace objects", () => { + assertStrictEquals(isModuleNamespaceObject(testModuleNamespaceOpbject), true); +}); + +test("Should return false for invalid module namespace objects", () => { + assertStrictEquals(isModuleNamespaceObject(assertStrictEquals), false); +}); + +// isNativeError +test("Should return true for valid Error types", () => { + assertStrictEquals(isNativeError(new Error()), true); + assertStrictEquals(isNativeError(new TypeError()), true); + assertStrictEquals(isNativeError(new RangeError()), true); +}); + +test("Should return false for invalid Error types", () => { + assertStrictEquals(isNativeError(null), false); + assertStrictEquals(isNativeError(NaN), false); +}); + +// isNumberObject +test("Should return true for valid number objects", () => { + assertStrictEquals(isNumberObject(new Number(0)), true); +}); + +test("Should return false for invalid number types", () => { + assertStrictEquals(isNumberObject(0), false); +}); + +// isBigIntObject +test("Should return true for valid number objects", () => { + assertStrictEquals(isBigIntObject(new Object(BigInt(42))), true); +}); + +test("Should return false for invalid number types", () => { + assertStrictEquals(isBigIntObject(BigInt(42)), false); +}); + +// isPromise +test("Should return true for valid Promise types", () => { + assertStrictEquals(isPromise(Promise.resolve(42)), true); +}); + +test("Should return false for invalid Promise types", () => { + assertStrictEquals(isPromise(new Object()), false); +}); + +// isRegExp +test("Should return true for valid RegExp", () => { + assertStrictEquals(isRegExp(/abc/), true); + assertStrictEquals(isRegExp(new RegExp("abc")), true); +}); + +test("Should return false for invalid RegExp types", () => { + assertStrictEquals(isRegExp({}), false); + assertStrictEquals(isRegExp("/abc/"), false); +}); + +// isSet +test("Should return true for valid Set types", () => { + assertStrictEquals(isSet(new Set()), true); +}); + +test("Should return false for invalid Set types", () => { + assertStrictEquals(isSet({}), false); + assertStrictEquals(isSet([]), false); + assertStrictEquals(isSet(new Map()), false); + assertStrictEquals(isSet(new Error()), false); +}); + +// isSetIterator +test("Should return true for valid Set Iterator types", () => { + const set = new Set(); + assertStrictEquals(isSetIterator(set.keys()), true); + assertStrictEquals(isSetIterator(set.values()), true); + assertStrictEquals(isSetIterator(set.entries()), true); + assertStrictEquals(isSetIterator(set[Symbol.iterator]()), true); +}); + +test("Should return false for invalid Set Iterator types", () => { + assertStrictEquals(isSetIterator(new Set()), false); + assertStrictEquals(isSetIterator([]), false); + assertStrictEquals(isSetIterator(new Map()), false); + assertStrictEquals(isSetIterator(new Error()), false); +}); + +// isSharedArrayBuffer +test("Should return true for valid SharedArrayBuffer types", () => { + assertStrictEquals(isSharedArrayBuffer(new SharedArrayBuffer(0)), true); +}); + +test("Should return false for invalid SharedArrayBuffer types", () => { + assertStrictEquals(isSharedArrayBuffer(new ArrayBuffer(0)), false); +}); + +// isStringObject +test("Should return true for valid String Object types", () => { + assertStrictEquals(isStringObject(new String("")), true); + assertStrictEquals(isStringObject(new String("Foo")), true); +}); + +test("Should return false for invalid String Object types", () => { + assertStrictEquals(isStringObject(""), false); + assertStrictEquals(isStringObject("Foo"), false); +}); + +// isSymbolObject +test("Should return true for valid Symbol types", () => { + assertStrictEquals(isSymbolObject(Object(Symbol("foo"))), true); +}); + +test("Should return false for invalid Symbol types", () => { + assertStrictEquals(isSymbolObject(Symbol("foo")), false); +}); + +// isTypedArray +test("Should return true for valid TypedArray types", () => { + assertStrictEquals(isTypedArray(new Uint8Array(0)), true); + assertStrictEquals(isTypedArray(new Float64Array(0)), true); +}); + +test("Should return false for invalid TypedArray types", () => { + assertStrictEquals(isTypedArray(new ArrayBuffer(0)), false); +}); + +// isUint8Array +test("Should return true for valid Uint8Array types", () => { + assertStrictEquals(isUint8Array(new Uint8Array(0)), true); +}); + +test("Should return false for invalid Uint8Array types", () => { + assertStrictEquals(isUint8Array(new ArrayBuffer(0)), false); + assertStrictEquals(isUint8Array(new Float64Array(0)), false); +}); + +// isUint8ClampedArray +test("Should return true for valid Uint8ClampedArray types", () => { + assertStrictEquals(isUint8ClampedArray(new Uint8ClampedArray(0)), true); +}); + +test("Should return false for invalid Uint8Array types", () => { + assertStrictEquals(isUint8ClampedArray(new ArrayBuffer(0)), false); + assertStrictEquals(isUint8ClampedArray(new Float64Array(0)), false); +}); + +// isUint16Array +test("Should return true for valid isUint16Array types", () => { + assertStrictEquals(isUint16Array(new Uint16Array(0)), true); +}); + +test("Should return false for invalid Uint16Array types", () => { + assertStrictEquals(isUint16Array(new ArrayBuffer(0)), false); + assertStrictEquals(isUint16Array(new Float64Array(0)), false); +}); + +// isUint32Array +test("Should return true for valid Uint32Array types", () => { + assertStrictEquals(isUint32Array(new Uint32Array(0)), true); +}); + +test("Should return false for invalid isUint16Array types", () => { + assertStrictEquals(isUint32Array(new ArrayBuffer(0)), false); + assertStrictEquals(isUint32Array(new Float64Array(0)), false); +}); + +// isWeakMap +test("Should return true for valid WeakMap types", () => { + assertStrictEquals(isWeakMap(new WeakMap()), true); +}); + +test("Should return false for invalid WeakMap types", () => { + assertStrictEquals(isWeakMap(new Set()), false); + assertStrictEquals(isWeakMap(new Map()), false); +}); + +// isWeakSet +test("Should return true for valid WeakSet types", () => { + assertStrictEquals(isWeakSet(new WeakSet()), true); +}); + +test("Should return false for invalid WeakSet types", () => { + assertStrictEquals(isWeakSet(new Set()), false); + assertStrictEquals(isWeakSet(new Map()), false); +}); diff --git a/std/node/buffer.ts b/std/node/buffer.ts new file mode 100644 index 00000000000000..fa911ee6f77c48 --- /dev/null +++ b/std/node/buffer.ts @@ -0,0 +1,417 @@ +/** + * See also https://nodejs.org/api/buffer.html + */ +export default class Buffer extends Uint8Array { + /** + * Allocates a new Buffer of size bytes. + */ + static alloc(size: number): Buffer { + return new Buffer(size); + } + + /** + * Returns the byte length of a string when encoded. This is not the same as + * String.prototype.length, which does not account for the encoding that is + * used to convert the string into bytes. + */ + static byteLength( + string: string | Buffer | ArrayBufferView | ArrayBuffer | SharedArrayBuffer + ): number { + if (typeof string != "string") return string.byteLength; + return new TextEncoder().encode(string).length; + } + + /** + * Returns a new Buffer which is the result of concatenating all the Buffer + * instances in the list together. + */ + static concat(list: Buffer[] | Uint8Array[], totalLength?: number): Buffer { + if (totalLength == undefined) { + totalLength = 0; + for (const buf of list) { + totalLength += buf.length; + } + } + + const buffer = new Buffer(totalLength); + let pos = 0; + for (const buf of list) { + buffer.set(buf, pos); + pos += buf.length; + } + + return buffer; + } + + /** + * Allocates a new Buffer using an array of bytes in the range 0 – 255. Array + * entries outside that range will be truncated to fit into it. + */ + static from(array: number[]): Buffer; + /** + * This creates a view of the ArrayBuffer without copying the underlying + * memory. For example, when passed a reference to the .buffer property of a + * TypedArray instance, the newly created Buffer will share the same allocated + * memory as the TypedArray. + */ + static from( + arrayBuffer: ArrayBuffer | SharedArrayBuffer, + byteOffset?: number, + length?: number + ): Buffer; + /** + * Copies the passed buffer data onto a new Buffer instance. + */ + static from(buffer: Buffer | Uint8Array): Buffer; + /** + * Creates a new Buffer containing string. + */ + static from(string: string): Buffer; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + static from(value: any, offset?: number, length?: number): Buffer { + if (typeof value == "string") + return new Buffer(new TextEncoder().encode(value).buffer); + + // workaround for https://github.com/microsoft/TypeScript/issues/38446 + return new Buffer(value, offset!, length); + } + + /** + * Returns true if obj is a Buffer, false otherwise. + */ + static isBuffer(obj: object): obj is Buffer { + return obj instanceof Buffer; + } + + /** + * Copies data from a region of buf to a region in target, even if the target + * memory region overlaps with buf. + */ + copy( + targetBuffer: Buffer | Uint8Array, + targetStart = 0, + sourceStart = 0, + sourceEnd = this.length + ): number { + const sourceBuffer = this.subarray(sourceStart, sourceEnd); + targetBuffer.set(sourceBuffer, targetStart); + return sourceBuffer.length; + } + + readBigInt64BE(offset = 0): bigint { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getBigInt64(offset); + } + readBigInt64LE(offset = 0): bigint { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getBigInt64(offset, true); + } + + readBigUInt64BE(offset = 0): bigint { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getBigUint64(offset); + } + readBigUInt64LE(offset = 0): bigint { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getBigUint64(offset, true); + } + + readDoubleBE(offset = 0): number { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getFloat64(offset); + } + readDoubleLE(offset = 0): number { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getFloat64(offset, true); + } + + readFloatBE(offset = 0): number { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getFloat32(offset); + } + readFloatLE(offset = 0): number { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getFloat32(offset, true); + } + + readInt8(offset = 0): number { + return new DataView(this.buffer, this.byteOffset, this.byteLength).getInt8( + offset + ); + } + + readInt16BE(offset = 0): number { + return new DataView(this.buffer, this.byteOffset, this.byteLength).getInt16( + offset + ); + } + readInt16LE(offset = 0): number { + return new DataView(this.buffer, this.byteOffset, this.byteLength).getInt16( + offset, + true + ); + } + + readInt32BE(offset = 0): number { + return new DataView(this.buffer, this.byteOffset, this.byteLength).getInt32( + offset + ); + } + readInt32LE(offset = 0): number { + return new DataView(this.buffer, this.byteOffset, this.byteLength).getInt32( + offset, + true + ); + } + + readUInt8(offset = 0): number { + return new DataView(this.buffer, this.byteOffset, this.byteLength).getUint8( + offset + ); + } + + readUInt16BE(offset = 0): number { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getUint16(offset); + } + readUInt16LE(offset = 0): number { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getUint16(offset, true); + } + + readUInt32BE(offset = 0): number { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getUint32(offset); + } + readUInt32LE(offset = 0): number { + return new DataView( + this.buffer, + this.byteOffset, + this.byteLength + ).getUint32(offset, true); + } + + /** + * Returns a new Buffer that references the same memory as the original, but + * offset and cropped by the start and end indices. + */ + slice(begin = 0, end = this.length): Buffer { + // workaround for https://github.com/microsoft/TypeScript/issues/38665 + return this.subarray(begin, end) as Buffer; + } + + /** + * Returns a JSON representation of buf. JSON.stringify() implicitly calls + * this function when stringifying a Buffer instance. + */ + toJSON(): object { + return { type: "Buffer", data: Array.from(this) }; + } + + /** + * Decodes buf to a string according to the specified character encoding in + * encoding. start and end may be passed to decode only a subset of buf. + */ + toString(encoding = "utf8", start = 0, end = this.length): string { + return new TextDecoder(encoding).decode(this.subarray(start, end)); + } + + /** + * Writes string to buf at offset according to the character encoding in + * encoding. The length parameter is the number of bytes to write. If buf did + * not contain enough space to fit the entire string, only part of string will + * be written. However, partially encoded characters will not be written. + */ + write(string: string, offset = 0, length = this.length): number { + return new TextEncoder().encodeInto( + string, + this.subarray(offset, offset + length) + ).written; + } + + writeBigInt64BE(value: bigint, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setBigInt64( + offset, + value + ); + return offset + 4; + } + writeBigInt64LE(value: bigint, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setBigInt64( + offset, + value, + true + ); + return offset + 4; + } + + writeBigUInt64BE(value: bigint, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setBigUint64( + offset, + value + ); + return offset + 4; + } + writeBigUInt64LE(value: bigint, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setBigUint64( + offset, + value, + true + ); + return offset + 4; + } + + writeDoubleBE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setFloat64( + offset, + value + ); + return offset + 8; + } + writeDoubleLE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setFloat64( + offset, + value, + true + ); + return offset + 8; + } + + writeFloatBE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setFloat32( + offset, + value + ); + return offset + 4; + } + writeFloatLE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setFloat32( + offset, + value, + true + ); + return offset + 4; + } + + writeInt8(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setInt8( + offset, + value + ); + return offset + 1; + } + + writeInt16BE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setInt16( + offset, + value + ); + return offset + 2; + } + writeInt16LE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setInt16( + offset, + value, + true + ); + return offset + 2; + } + + writeInt32BE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setUint32( + offset, + value + ); + return offset + 4; + } + writeInt32LE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setInt32( + offset, + value, + true + ); + return offset + 4; + } + + writeUInt8(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setUint8( + offset, + value + ); + return offset + 1; + } + + writeUInt16BE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setUint16( + offset, + value + ); + return offset + 2; + } + writeUInt16LE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setUint16( + offset, + value, + true + ); + return offset + 2; + } + + writeUInt32BE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setUint32( + offset, + value + ); + return offset + 4; + } + writeUInt32LE(value: number, offset = 0): number { + new DataView(this.buffer, this.byteOffset, this.byteLength).setUint32( + offset, + value, + true + ); + return offset + 4; + } +} + +export { Buffer }; + +Object.defineProperty(globalThis, "Buffer", { + value: Buffer, + enumerable: false, + writable: true, + configurable: true, +}); diff --git a/std/node/events.ts b/std/node/events.ts index aa62e4d432360a..cb9acb0f5a09fe 100644 --- a/std/node/events.ts +++ b/std/node/events.ts @@ -22,7 +22,7 @@ // USE OR OTHER DEALINGS IN THE SOFTWARE. import { validateIntegerRange } from "./util.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; export interface WrappedFunction extends Function { listener: Function; diff --git a/std/node/global.ts b/std/node/global.ts index c877d1d531b3c3..0cb6b8b06bfa27 100644 --- a/std/node/global.ts +++ b/std/node/global.ts @@ -5,5 +5,5 @@ Object.defineProperty(globalThis, Symbol.toStringTag, { configurable: true, }); -// @ts-ignore -globalThis["global"] = globalThis; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +(globalThis as any)["global"] = globalThis; diff --git a/std/node/module.ts b/std/node/module.ts index 4e0c55c1d84a6f..daa01d2b24043d 100644 --- a/std/node/module.ts +++ b/std/node/module.ts @@ -21,6 +21,7 @@ import "./global.ts"; +import * as nodeBuffer from "./buffer.ts"; import * as nodeFS from "./fs.ts"; import * as nodeUtil from "./util.ts"; import * as nodePath from "./path.ts"; @@ -30,7 +31,7 @@ import * as nodeEvents from "./events.ts"; import * as nodeQueryString from "./querystring.ts"; import * as path from "../path/mod.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; import { pathToFileURL, fileURLToPath } from "./url.ts"; const CHAR_FORWARD_SLASH = "/".charCodeAt(0); @@ -262,10 +263,11 @@ class Module { if (requireStack.length > 0) { message = message + "\nRequire stack:\n- " + requireStack.join("\n- "); } - const err = new Error(message); - // @ts-ignore + const err = new Error(message) as Error & { + code: string; + requireStack: string[]; + }; err.code = "MODULE_NOT_FOUND"; - // @ts-ignore err.requireStack = requireStack; throw err; } @@ -594,6 +596,7 @@ function createNativeModule(id: string, exports: any): Module { return mod; } +nativeModulePolyfill.set("buffer", createNativeModule("buffer", nodeBuffer)); nativeModulePolyfill.set("fs", createNativeModule("fs", nodeFS)); nativeModulePolyfill.set("events", createNativeModule("events", nodeEvents)); nativeModulePolyfill.set("os", createNativeModule("os", nodeOs)); @@ -732,12 +735,10 @@ function tryPackage( if (actual === false) { actual = tryExtensions(path.resolve(requestPath, "index"), exts, isMain); if (!actual) { - // eslint-disable-next-line no-restricted-syntax const err = new Error( `Cannot find module '${filename}'. ` + 'Please verify that the package.json has a valid "main" entry' - ); - // @ts-ignore + ) as Error & { code: string }; err.code = "MODULE_NOT_FOUND"; throw err; } @@ -881,8 +882,7 @@ function applyExports(basePath: string, expansion: string): string { const e = new Error( `Package exports for '${basePath}' do not define ` + `a '${mappingKey}' subpath` - ); - // @ts-ignore + ) as Error & { code?: string }; e.code = "MODULE_NOT_FOUND"; throw e; } @@ -973,7 +973,7 @@ function resolveExportsTarget( } } } - let e: Error; + let e: Error & { code?: string }; if (mappingKey !== ".") { e = new Error( `Package exports for '${basePath}' do not define a ` + @@ -982,7 +982,6 @@ function resolveExportsTarget( } else { e = new Error(`No valid exports main found for '${basePath}'`); } - // @ts-ignore e.code = "MODULE_NOT_FOUND"; throw e; } @@ -1006,8 +1005,7 @@ const CircularRequirePrototypeWarningProxy = new Proxy( {}, { // eslint-disable-next-line @typescript-eslint/no-explicit-any - get(target, prop): any { - // @ts-ignore + get(target: Record, prop: string): any { if (prop in target) return target[prop]; emitCircularRequireWarning(prop); return undefined; @@ -1058,8 +1056,8 @@ type RequireWrapper = ( function wrapSafe(filename: string, content: string): RequireWrapper { // TODO: fix this const wrapper = Module.wrap(content); - // @ts-ignore - const [f, err] = Deno.core.evalContext(wrapper, filename); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const [f, err] = (Deno as any).core.evalContext(wrapper, filename); if (err) { throw err; } diff --git a/std/node/module_test.ts b/std/node/module_test.ts index 234e9f0f420971..b1a22c0f646a9f 100644 --- a/std/node/module_test.ts +++ b/std/node/module_test.ts @@ -1,7 +1,11 @@ /* eslint-disable @typescript-eslint/no-var-requires */ const { test } = Deno; -import { assertEquals, assert, assertStrContains } from "../testing/asserts.ts"; +import { + assertEquals, + assert, + assertStringContains, +} from "../testing/asserts.ts"; import { createRequire } from "./module.ts"; const require = createRequire(import.meta.url); @@ -54,6 +58,6 @@ test("requireStack", function () { try { hello(); } catch (e) { - assertStrContains(e.stack, "/tests/cjs/cjs_throw.js"); + assertStringContains(e.stack, "/tests/cjs/cjs_throw.js"); } }); diff --git a/std/node/util.ts b/std/node/util.ts index 8c3fdabe877491..b115b061bc2b9a 100644 --- a/std/node/util.ts +++ b/std/node/util.ts @@ -1,4 +1,7 @@ export { callbackify } from "./_util/_util_callbackify.ts"; +import * as types from "./_util/_util_types.ts"; + +export { types }; export function isArray(value: unknown): boolean { return Array.isArray(value); diff --git a/std/node/util_test.ts b/std/node/util_test.ts index 7036368b622a1d..b6439644101c43 100644 --- a/std/node/util_test.ts +++ b/std/node/util_test.ts @@ -166,3 +166,11 @@ test({ assert(te instanceof TextEncoder); }, }); + +test({ + name: "[util] isDate", + fn() { + // Test verifies the method is exposed. See _util/_util_types_test for details + assert(util.types.isDate(new Date())); + }, +}); diff --git a/std/path/_constants.ts b/std/path/_constants.ts index ae0aac18465c0d..186c32ab532635 100644 --- a/std/path/_constants.ts +++ b/std/path/_constants.ts @@ -1,7 +1,6 @@ // Copyright the Browserify authors. MIT License. // Ported from https://github.com/browserify/path-browserify/ - -const { build } = Deno; +/** This module is browser compatible. */ // Alphabet chars. export const CHAR_UPPERCASE_A = 65; /* A */ @@ -48,7 +47,14 @@ export const CHAR_EQUAL = 61; /* = */ export const CHAR_0 = 48; /* 0 */ export const CHAR_9 = 57; /* 9 */ -const isWindows = build.os == "windows"; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const navigator = (globalThis as any).navigator; + +let isWindows = false; +if (globalThis.Deno != null) { + isWindows = Deno.build.os == "windows"; +} else if (navigator?.appVersion != null) { + isWindows = navigator.appVersion.includes("Win"); +} -export const SEP = isWindows ? "\\" : "/"; -export const SEP_PATTERN = isWindows ? /[\\/]+/ : /\/+/; +export { isWindows }; diff --git a/std/path/_globrex.ts b/std/path/_globrex.ts index 2b3af14ad0b333..6ad297d8643df5 100644 --- a/std/path/_globrex.ts +++ b/std/path/_globrex.ts @@ -1,8 +1,10 @@ // This file is ported from globrex@0.1.2 // MIT License // Copyright (c) 2018 Terkel Gjervig Nielsen +/** This module is browser compatible. */ + +import { isWindows as isWin } from "./_constants.ts"; -const isWin = Deno.build.os === "windows"; const SEP = isWin ? `(?:\\\\|\\/)` : `\\/`; const SEP_ESC = isWin ? `\\\\` : `/`; const SEP_RAW = isWin ? `\\` : `/`; diff --git a/std/path/interface.ts b/std/path/_interface.ts similarity index 93% rename from std/path/interface.ts rename to std/path/_interface.ts index b31c89ea732062..6c82c9c35cf452 100644 --- a/std/path/interface.ts +++ b/std/path/_interface.ts @@ -1,3 +1,5 @@ +/** This module is browser compatible. */ + /** * A parsed path object generated by path.parse() or consumed by path.format(). */ diff --git a/std/path/_util.ts b/std/path/_util.ts index 2776303cb6ed06..8ae40373b21971 100644 --- a/std/path/_util.ts +++ b/std/path/_util.ts @@ -1,7 +1,8 @@ // Copyright the Browserify authors. MIT License. // Ported from https://github.com/browserify/path-browserify/ +/** This module is browser compatible. */ -import { FormatInputPathObject } from "./interface.ts"; +import { FormatInputPathObject } from "./_interface.ts"; import { CHAR_UPPERCASE_A, CHAR_LOWERCASE_A, diff --git a/std/path/common.ts b/std/path/common.ts index e0e51ef239108f..01470105ca91a2 100644 --- a/std/path/common.ts +++ b/std/path/common.ts @@ -1,4 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +/** This module is browser compatible. */ import { SEP } from "./separator.ts"; diff --git a/std/path/glob.ts b/std/path/glob.ts index 80672579d2c684..c7d23b3441b1ae 100644 --- a/std/path/glob.ts +++ b/std/path/glob.ts @@ -1,7 +1,10 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +/** This module is browser compatible. */ + import { SEP, SEP_PATTERN } from "./separator.ts"; import { globrex } from "./_globrex.ts"; import { join, normalize } from "./mod.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; export interface GlobOptions { extended?: boolean; diff --git a/std/path/mod.ts b/std/path/mod.ts index 9cb7f1edb2fa2e..0b4156e69981eb 100644 --- a/std/path/mod.ts +++ b/std/path/mod.ts @@ -1,11 +1,11 @@ // Copyright the Browserify authors. MIT License. // Ported mostly from https://github.com/browserify/path-browserify/ +/** This module is browser compatible. */ +import { isWindows } from "./_constants.ts"; import * as _win32 from "./win32.ts"; import * as _posix from "./posix.ts"; -const isWindows = Deno.build.os == "windows"; - const path = isWindows ? _win32 : _posix; export const win32 = _win32; @@ -29,5 +29,5 @@ export const { export * from "./common.ts"; export { SEP, SEP_PATTERN } from "./separator.ts"; -export * from "./interface.ts"; +export * from "./_interface.ts"; export * from "./glob.ts"; diff --git a/std/path/posix.ts b/std/path/posix.ts index e88eb3f973ea17..365232e336f684 100644 --- a/std/path/posix.ts +++ b/std/path/posix.ts @@ -1,8 +1,8 @@ // Copyright the Browserify authors. MIT License. // Ported from https://github.com/browserify/path-browserify/ +/** This module is browser compatible. */ -const { cwd } = Deno; -import { FormatInputPathObject, ParsedPath } from "./interface.ts"; +import { FormatInputPathObject, ParsedPath } from "./_interface.ts"; import { CHAR_DOT, CHAR_FORWARD_SLASH } from "./_constants.ts"; import { @@ -24,7 +24,12 @@ export function resolve(...pathSegments: string[]): string { let path: string; if (i >= 0) path = pathSegments[i]; - else path = cwd(); + else { + if (globalThis.Deno == null) { + throw new TypeError("Resolved a relative path without a CWD."); + } + path = Deno.cwd(); + } assertPath(path); diff --git a/std/path/separator.ts b/std/path/separator.ts index fb990b8086d002..4b54ad4384b4a8 100644 --- a/std/path/separator.ts +++ b/std/path/separator.ts @@ -1,4 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -const isWindows = Deno.build.os == "windows"; +/** This module is browser compatible. */ + +import { isWindows } from "./_constants.ts"; + export const SEP = isWindows ? "\\" : "/"; export const SEP_PATTERN = isWindows ? /[\\/]+/ : /\/+/; diff --git a/std/path/win32.ts b/std/path/win32.ts index 0557b3768ee079..8e438a79c3f76e 100644 --- a/std/path/win32.ts +++ b/std/path/win32.ts @@ -1,8 +1,8 @@ // Copyright the Browserify authors. MIT License. // Ported from https://github.com/browserify/path-browserify/ +/** This module is browser compatible. */ -const { cwd, env } = Deno; -import { FormatInputPathObject, ParsedPath } from "./interface.ts"; +import { FormatInputPathObject, ParsedPath } from "./_interface.ts"; import { CHAR_DOT, CHAR_BACKWARD_SLASH, @@ -17,7 +17,7 @@ import { normalizeString, _format, } from "./_util.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; export const sep = "\\"; export const delimiter = ";"; @@ -32,14 +32,20 @@ export function resolve(...pathSegments: string[]): string { if (i >= 0) { path = pathSegments[i]; } else if (!resolvedDevice) { - path = cwd(); + if (globalThis.Deno == null) { + throw new TypeError("Resolved a drive-letter-less path without a CWD."); + } + path = Deno.cwd(); } else { + if (globalThis.Deno == null) { + throw new TypeError("Resolved a relative path without a CWD."); + } // Windows has the concept of drive-specific current working // directories. If we've resolved a drive letter but not yet an // absolute path, get cwd for that drive, or the process cwd if // the drive cwd is not available. We're sure the device is not // a UNC path at this points, because UNC paths are always absolute. - path = env.get(`=${resolvedDevice}`) || cwd(); + path = Deno.env.get(`=${resolvedDevice}`) || Deno.cwd(); // Verify that a cwd was found and that it actually points // to our drive. If not, default to the drive's root. diff --git a/std/signal/mod.ts b/std/signal/mod.ts index c15d1b3269ae45..f09f768824dc6a 100644 --- a/std/signal/mod.ts +++ b/std/signal/mod.ts @@ -61,12 +61,12 @@ export function signal( export function onSignal(signo: number, callback: () => void): Disposable { const sig = signal(signo); - //setTimeout allows `sig` to be returned before blocking on the await - setTimeout(async () => { + // allows `sig` to be returned before blocking on the await + (async (): Promise => { for await (const _ of sig) { callback(); } - }, 0); + })(); return sig; } diff --git a/std/signal/test.ts b/std/signal/test.ts index 15ebbdcc723c36..ef79a303bcf80f 100644 --- a/std/signal/test.ts +++ b/std/signal/test.ts @@ -9,8 +9,8 @@ test({ fn() { assertThrows( () => { - // @ts-ignore - signal(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (signal as any)(); }, Error, "No signals are given. You need to specify at least one signal to create a signal stream." diff --git a/std/testing/README.md b/std/testing/README.md index 66d80d4bc0991c..42ce49d5c8dbc4 100644 --- a/std/testing/README.md +++ b/std/testing/README.md @@ -137,9 +137,20 @@ Deno.test("fails", async function (): Promise { }); ``` -### Benching Usage +## Benching -Basic usage: +With this module you can benchmark your code and get information on how is it +performing. + +### Basic usage: + +Benchmarks can be registered using the `bench` function, where you can define a +code, that should be benchmarked. `b.start()` has to be called at the start of +the part you want to benchmark and `b.stop()` at the end of it, otherwise an +error will be thrown. + +After that simply calling `runBenchmarks()` will benchmark all registered +benchmarks and log the results in the commandline. ```ts import { runBenchmarks, bench } from "https://deno.land/std/testing/bench.ts"; @@ -167,47 +178,63 @@ bench({ }); ``` +Running specific benchmarks using regular expressions: + +```ts +runBenchmarks({ only: /desired/, skip: /exceptions/ }); +``` + +### Processing benchmark results + +`runBenchmarks()` returns a `Promise`, so you can process +the benchmarking results yourself. It contains detailed results of each +benchmark's run as `BenchmarkResult` s. + +```ts +runBenchmarks() + .then((results: BenchmarkRunResult) => { + console.log(results); + }) + .catch((error: Error) => { + // ... errors if benchmark was badly constructed + }); +``` + +### Processing benchmarking progress + +`runBenchmarks()` accepts an optional progress handler callback function, so you +can get information on the progress of the running benchmarking. + +Using `{ silent: true }` means you wont see the default progression logs in the +commandline. + +```ts +runBenchmarks({ silent: true }, (p: BenchmarkRunProgress) => { + // initial progress data + if (p.state === ProgressState.BenchmarkingStart) { + console.log( + `Starting benchmarking. Queued: ${p.queued.length}, filtered: ${p.filtered}` + ); + } + // ... +}); +``` + #### Benching API ##### `bench(benchmark: BenchmarkDefinition | BenchmarkFunction): void` Registers a benchmark that will be run once `runBenchmarks` is called. -##### `runBenchmarks(opts?: BenchmarkRunOptions): Promise` +##### `runBenchmarks(opts?: BenchmarkRunOptions, progressCb?: (p: BenchmarkRunProgress) => void): Promise` Runs all registered benchmarks serially. Filtering can be applied by setting `BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular -expressions matching benchmark names. - -##### `runIfMain(meta: ImportMeta, opts?: BenchmarkRunOptions): Promise` +expressions matching benchmark names. Default progression logs can be turned off +with the `BenchmarkRunOptions.silent` flag. -Runs specified benchmarks if the enclosing script is main. +##### `clearBenchmarks(opts?: BenchmarkClearOptions): void` -##### Other exports - -```ts -/** Provides methods for starting and stopping a benchmark clock. */ -export interface BenchmarkTimer { - start: () => void; - stop: () => void; -} - -/** Defines a benchmark through a named function. */ -export interface BenchmarkFunction { - (b: BenchmarkTimer): void | Promise; - name: string; -} - -/** Defines a benchmark definition with configurable runs. */ -export interface BenchmarkDefinition { - func: BenchmarkFunction; - name: string; - runs?: number; -} - -/** Defines runBenchmark's run constraints by matching benchmark names. */ -export interface BenchmarkRunOptions { - only?: RegExp; - skip?: RegExp; -} -``` +Clears all registered benchmarks, so calling `runBenchmarks()` after it wont run +them. Filtering can be applied by setting `BenchmarkRunOptions.only` and/or +`BenchmarkRunOptions.skip` to regular expressions matching benchmark names. diff --git a/std/testing/asserts.ts b/std/testing/asserts.ts index d3f8bb678497c4..5e1b8af692caa9 100644 --- a/std/testing/asserts.ts +++ b/std/testing/asserts.ts @@ -1,5 +1,8 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -import { red, green, white, gray, bold } from "../fmt/colors.ts"; +/** This module is browser compatible. Do not rely on good formatting of values + * for AssertionError messages in browsers. */ + +import { red, green, white, gray, bold, stripColor } from "../fmt/colors.ts"; import diff, { DiffType, DiffResult } from "./diff.ts"; const CAN_NOT_DISPLAY = "[Cannot display]"; @@ -17,7 +20,7 @@ export class AssertionError extends Error { } function format(v: unknown): string { - let string = Deno.inspect(v); + let string = globalThis.Deno ? Deno.inspect(v) : String(v); if (typeof v == "string") { string = `"${string.replace(/(?=["\\])/g, "\\")}"`; } @@ -201,7 +204,7 @@ export function assertNotEquals( * Make an assertion that `actual` and `expected` are strictly equal. If * not then throw. */ -export function assertStrictEq( +export function assertStrictEquals( actual: unknown, expected: unknown, msg?: string @@ -247,14 +250,14 @@ export function assertStrictEq( * Make an assertion that actual contains expected. If not * then thrown. */ -export function assertStrContains( +export function assertStringContains( actual: string, expected: string, msg?: string ): void { if (!actual.includes(expected)) { if (!msg) { - msg = `actual: "${actual}" expected to contains: "${expected}"`; + msg = `actual: "${actual}" expected to contain: "${expected}"`; } throw new AssertionError(msg); } @@ -286,9 +289,9 @@ export function assertArrayContains( return; } if (!msg) { - msg = `actual: "${actual}" expected to contains: "${expected}"`; - msg += "\n"; - msg += `missing: ${missing}`; + msg = `actual: "${format(actual)}" expected to contain: "${format( + expected + )}"\nmissing: ${format(missing)}`; } throw new AssertionError(msg); } @@ -322,8 +325,8 @@ export function fail(msg?: string): void { * throws. An error class and a string that should be included in the * error message can also be asserted. */ -export function assertThrows( - fn: () => void, +export function assertThrows( + fn: () => T, ErrorClass?: Constructor, msgIncludes = "", msg?: string @@ -339,7 +342,10 @@ export function assertThrows( }"${msg ? `: ${msg}` : "."}`; throw new AssertionError(msg); } - if (msgIncludes && !e.message.includes(msgIncludes)) { + if ( + msgIncludes && + !stripColor(e.message).includes(stripColor(msgIncludes)) + ) { msg = `Expected error message to include "${msgIncludes}", but got "${ e.message }"${msg ? `: ${msg}` : "."}`; @@ -355,8 +361,8 @@ export function assertThrows( return error; } -export async function assertThrowsAsync( - fn: () => Promise, +export async function assertThrowsAsync( + fn: () => Promise, ErrorClass?: Constructor, msgIncludes = "", msg?: string @@ -372,7 +378,10 @@ export async function assertThrowsAsync( }"${msg ? `: ${msg}` : "."}`; throw new AssertionError(msg); } - if (msgIncludes && !e.message.includes(msgIncludes)) { + if ( + msgIncludes && + !stripColor(e.message).includes(stripColor(msgIncludes)) + ) { msg = `Expected error message to include "${msgIncludes}", but got "${ e.message }"${msg ? `: ${msg}` : "."}`; diff --git a/std/testing/asserts_test.ts b/std/testing/asserts_test.ts index 14eabca610f741..feb4d097d6bb40 100644 --- a/std/testing/asserts_test.ts +++ b/std/testing/asserts_test.ts @@ -3,12 +3,13 @@ import { assert, assertNotEquals, - assertStrContains, + assertStringContains, assertArrayContains, assertMatch, assertEquals, - assertStrictEq, + assertStrictEquals, assertThrows, + assertThrowsAsync, AssertionError, equal, fail, @@ -132,12 +133,12 @@ test("testingNotEquals", function (): void { }); test("testingAssertStringContains", function (): void { - assertStrContains("Denosaurus", "saur"); - assertStrContains("Denosaurus", "Deno"); - assertStrContains("Denosaurus", "rus"); + assertStringContains("Denosaurus", "saur"); + assertStringContains("Denosaurus", "Deno"); + assertStringContains("Denosaurus", "rus"); let didThrow; try { - assertStrContains("Denosaurus", "Raptor"); + assertStringContains("Denosaurus", "Raptor"); didThrow = false; } catch (e) { assert(e instanceof AssertionError); @@ -151,25 +152,21 @@ test("testingArrayContains", function (): void { const fixtureObject = [{ deno: "luv" }, { deno: "Js" }]; assertArrayContains(fixture, ["deno"]); assertArrayContains(fixtureObject, [{ deno: "luv" }]); - let didThrow; - try { - assertArrayContains(fixtureObject, [{ deno: "node" }]); - didThrow = false; - } catch (e) { - assert(e instanceof AssertionError); - didThrow = true; - } - assertEquals(didThrow, true); + assertThrows( + (): void => assertArrayContains(fixtureObject, [{ deno: "node" }]), + AssertionError, + `actual: "[ { deno: "luv" }, { deno: "Js" } ]" expected to contain: "[ { deno: "node" } ]"\nmissing: [ { deno: "node" } ]` + ); }); test("testingAssertStringContainsThrow", function (): void { let didThrow = false; try { - assertStrContains("Denosaurus from Jurassic", "Raptor"); + assertStringContains("Denosaurus from Jurassic", "Raptor"); } catch (e) { assert( e.message === - `actual: "Denosaurus from Jurassic" expected to contains: "Raptor"` + `actual: "Denosaurus from Jurassic" expected to contain: "Raptor"` ); assert(e instanceof AssertionError); didThrow = true; @@ -249,6 +246,20 @@ test("testingAssertFailWithWrongErrorClass", function (): void { ); }); +test("testingAssertThrowsWithReturnType", () => { + assertThrows(() => { + throw new Error(); + return "a string"; + }); +}); + +test("testingAssertThrowsAsyncWithReturnType", () => { + assertThrowsAsync(() => { + throw new Error(); + return Promise.resolve("a Promise"); + }); +}); + const createHeader = (): string[] => [ "", "", @@ -347,17 +358,17 @@ test({ test({ name: "strict pass case", fn(): void { - assertStrictEq(true, true); - assertStrictEq(10, 10); - assertStrictEq("abc", "abc"); + assertStrictEquals(true, true); + assertStrictEquals(10, 10); + assertStrictEquals("abc", "abc"); const xs = [1, false, "foo"]; const ys = xs; - assertStrictEq(xs, ys); + assertStrictEquals(xs, ys); const x = { a: 1 }; const y = x; - assertStrictEq(x, y); + assertStrictEquals(x, y); }, }); @@ -365,7 +376,7 @@ test({ name: "strict failed with structure diff", fn(): void { assertThrows( - (): void => assertStrictEq({ a: 1, b: 2 }, { a: 1, c: [3] }), + (): void => assertStrictEquals({ a: 1, b: 2 }, { a: 1, c: [3] }), AssertionError, [ "Values are not strictly equal:", @@ -382,7 +393,7 @@ test({ name: "strict failed with reference diff", fn(): void { assertThrows( - (): void => assertStrictEq({ a: 1, b: 2 }, { a: 1, b: 2 }), + (): void => assertStrictEquals({ a: 1, b: 2 }, { a: 1, b: 2 }), AssertionError, [ "Values have the same structure but are not reference-equal:\n", diff --git a/std/testing/bench.ts b/std/testing/bench.ts index cd7b89e8c409a8..87be641abeaed9 100644 --- a/std/testing/bench.ts +++ b/std/testing/bench.ts @@ -1,10 +1,12 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +import { deepAssign } from "../_util/deep_assign.ts"; -const { exit, noColor } = Deno; +const { noColor } = Deno; interface BenchmarkClock { start: number; stop: number; + for?: string; } /** Provides methods for starting and stopping a benchmark clock. */ @@ -23,13 +25,76 @@ export interface BenchmarkFunction { export interface BenchmarkDefinition { func: BenchmarkFunction; name: string; + /** Defines how many times the provided `func` should be benchmarked in succession */ runs?: number; } /** Defines runBenchmark's run constraints by matching benchmark names. */ export interface BenchmarkRunOptions { + /** Only benchmarks which name match this regexp will be run*/ only?: RegExp; + /** Benchmarks which name match this regexp will be skipped */ skip?: RegExp; + /** Setting it to true prevents default benchmarking progress logs to the commandline*/ + silent?: boolean; +} + +/** Defines clearBenchmark's constraints by matching benchmark names. */ +export interface BenchmarkClearOptions { + /** Only benchmarks which name match this regexp will be removed */ + only?: RegExp; + /** Benchmarks which name match this regexp will be kept */ + skip?: RegExp; +} + +/** Defines the result of a single benchmark */ +export interface BenchmarkResult { + /** The name of the benchmark */ + name: string; + /** The total time it took to run a given bechmark */ + totalMs: number; + /** Times the benchmark was run in succession. */ + runsCount: number; + /** The average time of running the benchmark in milliseconds. */ + measuredRunsAvgMs: number; + /** The individual measurements in milliseconds it took to run the benchmark.*/ + measuredRunsMs: number[]; +} + +/** Defines the result of a `runBenchmarks` call */ +export interface BenchmarkRunResult { + /** How many benchmark were ignored by the provided `only` and `skip` */ + filtered: number; + /** The individual results for each benchmark that was run */ + results: BenchmarkResult[]; +} + +/** Defines the current progress during the run of `runBenchmarks` */ +export interface BenchmarkRunProgress extends BenchmarkRunResult { + /** List of the queued benchmarks to run with their name and their run count */ + queued: Array<{ name: string; runsCount: number }>; + /** The currently running benchmark with its name, run count and the already finished measurements in milliseconds */ + running?: { name: string; runsCount: number; measuredRunsMs: number[] }; + /** Indicates in which state benchmarking currently is */ + state: ProgressState; +} + +/** Defines the states `BenchmarkRunProgress` can be in */ +export enum ProgressState { + BenchmarkingStart = "benchmarking_start", + BenchStart = "bench_start", + BenchPartialResult = "bench_partial_result", + BenchResult = "bench_result", + BenchmarkingEnd = "benchmarking_end", +} + +export class BenchmarkRunError extends Error { + benchmarkName?: string; + constructor(msg: string, benchmarkName?: string) { + super(msg); + this.name = "BenchmarkRunError"; + this.benchmarkName = benchmarkName; + } } function red(text: string): string { @@ -47,13 +112,19 @@ function verifyOr1Run(runs?: number): number { function assertTiming(clock: BenchmarkClock): void { // NaN indicates that a benchmark has not been timed properly if (!clock.stop) { - throw new Error("The benchmark timer's stop method must be called"); + throw new BenchmarkRunError( + `Running benchmarks FAILED during benchmark named [${clock.for}]. The benchmark timer's stop method must be called`, + clock.for + ); } else if (!clock.start) { - throw new Error("The benchmark timer's start method must be called"); + throw new BenchmarkRunError( + `Running benchmarks FAILED during benchmark named [${clock.for}]. The benchmark timer's start method must be called`, + clock.for + ); } else if (clock.start > clock.stop) { - throw new Error( - "The benchmark timer's start method must be called before its " + - "stop method" + throw new BenchmarkRunError( + `Running benchmarks FAILED during benchmark named [${clock.for}]. The benchmark timer's start method must be called before its stop method`, + clock.for ); } } @@ -64,6 +135,12 @@ function createBenchmarkTimer(clock: BenchmarkClock): BenchmarkTimer { clock.start = performance.now(); }, stop(): void { + if (isNaN(clock.start)) { + throw new BenchmarkRunError( + `Running benchmarks FAILED during benchmark named [${clock.for}]. The benchmark timer's start method must be called before its stop method`, + clock.for + ); + } clock.stop = performance.now(); }, }; @@ -89,92 +166,191 @@ export function bench( } } -/** Runs all registered and non-skipped benchmarks serially. */ -export async function runBenchmarks({ +/** Clears benchmark candidates which name matches `only` and doesn't match `skip`. + * Removes all candidates if options were not provided */ +export function clearBenchmarks({ only = /[^\s]/, - skip = /^\s*$/, -}: BenchmarkRunOptions = {}): Promise { + skip = /$^/, +}: BenchmarkClearOptions = {}): void { + const keep = candidates.filter( + ({ name }): boolean => !only.test(name) || skip.test(name) + ); + candidates.splice(0, candidates.length); + candidates.push(...keep); +} + +/** + * Runs all registered and non-skipped benchmarks serially. + * + * @param [progressCb] provides the possibility to get updates of the current progress during the run of the benchmarking + * @returns results of the benchmarking + */ +export async function runBenchmarks( + { only = /[^\s]/, skip = /^\s*$/, silent }: BenchmarkRunOptions = {}, + progressCb?: (progress: BenchmarkRunProgress) => void +): Promise { // Filtering candidates by the "only" and "skip" constraint const benchmarks: BenchmarkDefinition[] = candidates.filter( ({ name }): boolean => only.test(name) && !skip.test(name) ); // Init main counters and error flag const filtered = candidates.length - benchmarks.length; - let measured = 0; - let failed = false; + let failError: Error | undefined = undefined; // Setting up a shared benchmark clock and timer const clock: BenchmarkClock = { start: NaN, stop: NaN }; const b = createBenchmarkTimer(clock); + + // Init progress data + const progress: BenchmarkRunProgress = { + // bench.run is already ensured with verifyOr1Run on register + queued: benchmarks.map((bench) => ({ + name: bench.name, + runsCount: bench.runs!, + })), + results: [], + filtered, + state: ProgressState.BenchmarkingStart, + }; + + // Publish initial progress data + publishProgress(progress, ProgressState.BenchmarkingStart, progressCb); + + if (!silent) { + console.log( + "running", + benchmarks.length, + `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}` + ); + } + // Iterating given benchmark definitions (await-in-loop) - console.log( - "running", - benchmarks.length, - `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}` - ); for (const { name, runs = 0, func } of benchmarks) { - // See https://github.com/denoland/deno/pull/1452 about groupCollapsed - console.groupCollapsed(`benchmark ${name} ... `); + if (!silent) { + // See https://github.com/denoland/deno/pull/1452 about groupCollapsed + console.groupCollapsed(`benchmark ${name} ... `); + } + + // Provide the benchmark name for clock assertions + clock.for = name; + + // Remove benchmark from queued + const queueIndex = progress.queued.findIndex( + (queued) => queued.name === name && queued.runsCount === runs + ); + if (queueIndex != -1) { + progress.queued.splice(queueIndex, 1); + } + // Init the progress of the running benchmark + progress.running = { name, runsCount: runs, measuredRunsMs: [] }; + // Publish starting of a benchmark + publishProgress(progress, ProgressState.BenchStart, progressCb); + // Trying benchmark.func let result = ""; try { - if (runs === 1) { + // Averaging runs + let pendingRuns = runs; + let totalMs = 0; + + // Would be better 2 not run these serially + while (true) { // b is a benchmark timer interfacing an unset (NaN) benchmark clock await func(b); // Making sure the benchmark was started/stopped properly assertTiming(clock); - result = `${clock.stop - clock.start}ms`; - } else if (runs > 1) { - // Averaging runs - let pendingRuns = runs; - let totalMs = 0; - // Would be better 2 not run these serially - while (true) { - // b is a benchmark timer interfacing an unset (NaN) benchmark clock - await func(b); - // Making sure the benchmark was started/stopped properly - assertTiming(clock); - // Summing up - totalMs += clock.stop - clock.start; - // Resetting the benchmark clock - clock.start = clock.stop = NaN; - // Once all ran - if (!--pendingRuns) { - result = `${runs} runs avg: ${totalMs / runs}ms`; - break; - } + + // Calculate length of run + const measuredMs = clock.stop - clock.start; + + // Summing up + totalMs += measuredMs; + // Adding partial result + progress.running.measuredRunsMs.push(measuredMs); + // Publish partial benchmark results + publishProgress(progress, ProgressState.BenchPartialResult, progressCb); + + // Resetting the benchmark clock + clock.start = clock.stop = NaN; + // Once all ran + if (!--pendingRuns) { + result = + runs == 1 + ? `${totalMs}ms` + : `${runs} runs avg: ${totalMs / runs}ms`; + // Adding results + progress.results.push({ + name, + totalMs, + runsCount: runs, + measuredRunsAvgMs: totalMs / runs, + measuredRunsMs: progress.running.measuredRunsMs, + }); + // Clear currently running + delete progress.running; + // Publish results of the benchmark + publishProgress(progress, ProgressState.BenchResult, progressCb); + break; } } } catch (err) { - failed = true; - console.groupEnd(); - console.error(red(err.stack)); + failError = err; + + if (!silent) { + console.groupEnd(); + console.error(red(err.stack)); + } + break; } - // Reporting - console.log(blue(result)); - console.groupEnd(); - measured++; + + if (!silent) { + // Reporting + console.log(blue(result)); + console.groupEnd(); + } + // Resetting the benchmark clock clock.start = clock.stop = NaN; + delete clock.for; } - // Closing results - console.log( - `benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` + - `${measured} measured; ${filtered} filtered` - ); - // Making sure the program exit code is not zero in case of failure - if (failed) { - setTimeout((): void => exit(1), 0); + + // Indicate finished running + delete progress.queued; + // Publish final result in Cb too + publishProgress(progress, ProgressState.BenchmarkingEnd, progressCb); + + if (!silent) { + // Closing results + console.log( + `benchmark result: ${!!failError ? red("FAIL") : blue("DONE")}. ` + + `${progress.results.length} measured; ${filtered} filtered` + ); } -} -/** Runs specified benchmarks if the enclosing script is main. */ -export function runIfMain( - meta: ImportMeta, - opts: BenchmarkRunOptions = {} -): Promise { - if (meta.main) { - return runBenchmarks(opts); + // Throw error if there was a failing benchmark + if (!!failError) { + throw failError; } - return Promise.resolve(undefined); + + const benchmarkRunResult = { + filtered, + results: progress.results, + }; + + return benchmarkRunResult; +} + +function publishProgress( + progress: BenchmarkRunProgress, + state: ProgressState, + progressCb?: (progress: BenchmarkRunProgress) => void +): void { + progressCb && progressCb(cloneProgressWithState(progress, state)); +} + +function cloneProgressWithState( + progress: BenchmarkRunProgress, + state: ProgressState +): BenchmarkRunProgress { + return deepAssign({}, progress, { state }) as BenchmarkRunProgress; } diff --git a/std/testing/bench_example.ts b/std/testing/bench_example.ts index 401516cca8de19..366521f85c440e 100644 --- a/std/testing/bench_example.ts +++ b/std/testing/bench_example.ts @@ -1,5 +1,5 @@ // https://deno.land/std/testing/bench.ts -import { BenchmarkTimer, bench, runIfMain } from "./bench.ts"; +import { BenchmarkTimer, bench, runBenchmarks } from "./bench.ts"; // Basic bench(function forIncrementX1e9(b: BenchmarkTimer): void { @@ -26,4 +26,6 @@ bench(function throwing(b): void { }); // Bench control -runIfMain(import.meta, { skip: /throw/ }); +if (import.meta.main) { + runBenchmarks({ skip: /throw/ }); +} diff --git a/std/testing/bench_test.ts b/std/testing/bench_test.ts index 6dfc18b1090974..a56d4fd5550cf1 100644 --- a/std/testing/bench_test.ts +++ b/std/testing/bench_test.ts @@ -1,7 +1,18 @@ const { test } = Deno; -import { bench, runBenchmarks } from "./bench.ts"; - -import "./bench_example.ts"; +import { + bench, + runBenchmarks, + BenchmarkRunError, + clearBenchmarks, + BenchmarkRunProgress, + ProgressState, +} from "./bench.ts"; +import { + assertEquals, + assert, + assertThrows, + assertThrowsAsync, +} from "./asserts.ts"; test({ name: "benching", @@ -57,6 +68,283 @@ test({ // Throws bc the timer's stop method is never called }); - await runBenchmarks({ skip: /throw/ }); + const benchResult = await runBenchmarks({ skip: /throw/ }); + + assertEquals(benchResult.filtered, 1); + assertEquals(benchResult.results.length, 5); + + const resultWithSingleRunsFiltered = benchResult.results.filter( + ({ name }) => name === "forDecrementX1e9" + ); + assertEquals(resultWithSingleRunsFiltered.length, 1); + + const resultWithSingleRuns = resultWithSingleRunsFiltered[0]; + assert(!!resultWithSingleRuns.runsCount); + assert(!!resultWithSingleRuns.measuredRunsAvgMs); + assert(!!resultWithSingleRuns.measuredRunsMs); + assertEquals(resultWithSingleRuns.runsCount, 1); + assertEquals(resultWithSingleRuns.measuredRunsMs.length, 1); + + const resultWithMultipleRunsFiltered = benchResult.results.filter( + ({ name }) => name === "runs100ForIncrementX1e6" + ); + assertEquals(resultWithMultipleRunsFiltered.length, 1); + + const resultWithMultipleRuns = resultWithMultipleRunsFiltered[0]; + assert(!!resultWithMultipleRuns.runsCount); + assert(!!resultWithMultipleRuns.measuredRunsAvgMs); + assert(!!resultWithMultipleRuns.measuredRunsMs); + assertEquals(resultWithMultipleRuns.runsCount, 100); + assertEquals(resultWithMultipleRuns.measuredRunsMs.length, 100); + + clearBenchmarks(); + }, +}); + +test({ + name: "Bench without name should throw", + fn() { + assertThrows( + (): void => { + bench(() => {}); + }, + Error, + "The benchmark function must not be anonymous" + ); + }, +}); + +test({ + name: "Bench without stop should throw", + fn: async function (): Promise { + await assertThrowsAsync( + async (): Promise => { + bench(function benchWithoutStop(b): void { + b.start(); + // Throws bc the timer's stop method is never called + }); + await runBenchmarks({ only: /benchWithoutStop/, silent: true }); + }, + BenchmarkRunError, + "The benchmark timer's stop method must be called" + ); + }, +}); + +test({ + name: "Bench without start should throw", + fn: async function (): Promise { + await assertThrowsAsync( + async (): Promise => { + bench(function benchWithoutStart(b): void { + b.stop(); + // Throws bc the timer's start method is never called + }); + await runBenchmarks({ only: /benchWithoutStart/, silent: true }); + }, + BenchmarkRunError, + "The benchmark timer's start method must be called" + ); }, }); + +test({ + name: "Bench with stop before start should throw", + fn: async function (): Promise { + await assertThrowsAsync( + async (): Promise => { + bench(function benchStopBeforeStart(b): void { + b.stop(); + b.start(); + // Throws bc the timer's stop is called before start + }); + await runBenchmarks({ only: /benchStopBeforeStart/, silent: true }); + }, + BenchmarkRunError, + "The benchmark timer's start method must be called before its stop method" + ); + }, +}); + +test({ + name: "clearBenchmarks should clear all candidates", + fn: async function (): Promise { + dummyBench("test"); + + clearBenchmarks(); + const benchingResults = await runBenchmarks({ silent: true }); + + assertEquals(benchingResults.filtered, 0); + assertEquals(benchingResults.results.length, 0); + }, +}); + +test({ + name: "clearBenchmarks with only as option", + fn: async function (): Promise { + // to reset candidates + clearBenchmarks(); + + dummyBench("test"); + dummyBench("onlyclear"); + + clearBenchmarks({ only: /only/ }); + const benchingResults = await runBenchmarks({ silent: true }); + + assertEquals(benchingResults.filtered, 0); + assertEquals(benchingResults.results.length, 1); + assertEquals(benchingResults.results[0].name, "test"); + }, +}); + +test({ + name: "clearBenchmarks with skip as option", + fn: async function (): Promise { + // to reset candidates + clearBenchmarks(); + + dummyBench("test"); + dummyBench("skipclear"); + + clearBenchmarks({ skip: /skip/ }); + const benchingResults = await runBenchmarks({ silent: true }); + + assertEquals(benchingResults.filtered, 0); + assertEquals(benchingResults.results.length, 1); + assertEquals(benchingResults.results[0].name, "skipclear"); + }, +}); + +test({ + name: "clearBenchmarks with only and skip as option", + fn: async function (): Promise { + // to reset candidates + clearBenchmarks(); + + dummyBench("test"); + dummyBench("clearonly"); + dummyBench("clearskip"); + dummyBench("clearonly"); + + clearBenchmarks({ only: /clear/, skip: /skip/ }); + const benchingResults = await runBenchmarks({ silent: true }); + + assertEquals(benchingResults.filtered, 0); + assertEquals(benchingResults.results.length, 2); + assert(!!benchingResults.results.find(({ name }) => name === "test")); + assert(!!benchingResults.results.find(({ name }) => name === "clearskip")); + }, +}); + +test({ + name: "progressCallback of runBenchmarks", + fn: async function (): Promise { + clearBenchmarks(); + dummyBench("skip"); + dummyBench("single"); + dummyBench("multiple", 2); + + const progressCallbacks: BenchmarkRunProgress[] = []; + + const benchingResults = await runBenchmarks( + { skip: /skip/, silent: true }, + (progress) => { + progressCallbacks.push(progress); + } + ); + + let pc = 0; + // Assert initial progress before running + let progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchmarkingStart); + assertEquals(progress.filtered, 1); + assertEquals(progress.queued.length, 2); + assertEquals(progress.running, undefined); + assertEquals(progress.results, []); + + // Assert start of bench "single" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchStart); + assertEquals(progress.filtered, 1); + assertEquals(progress.queued.length, 1); + assert(!!progress.queued.find(({ name }) => name == "multiple")); + assertEquals(progress.running, { + name: "single", + runsCount: 1, + measuredRunsMs: [], + }); + assertEquals(progress.results, []); + + // Assert running result of bench "single" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchPartialResult); + assertEquals(progress.queued.length, 1); + assertEquals(progress.running!.measuredRunsMs.length, 1); + assertEquals(progress.results.length, 0); + + // Assert result of bench "single" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchResult); + assertEquals(progress.queued.length, 1); + assertEquals(progress.running, undefined); + assertEquals(progress.results.length, 1); + assert(!!progress.results.find(({ name }) => name == "single")); + + // Assert start of bench "multiple" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchStart); + assertEquals(progress.queued.length, 0); + assertEquals(progress.running, { + name: "multiple", + runsCount: 2, + measuredRunsMs: [], + }); + assertEquals(progress.results.length, 1); + + // Assert first result of bench "multiple" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchPartialResult); + assertEquals(progress.queued.length, 0); + assertEquals(progress.running!.measuredRunsMs.length, 1); + assertEquals(progress.results.length, 1); + + // Assert second result of bench "multiple" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchPartialResult); + assertEquals(progress.queued.length, 0); + assertEquals(progress.running!.measuredRunsMs.length, 2); + assertEquals(progress.results.length, 1); + + // Assert finish of bench "multiple" + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchResult); + assertEquals(progress.queued.length, 0); + assertEquals(progress.running, undefined); + assertEquals(progress.results.length, 2); + assert(!!progress.results.find(({ name }) => name == "single")); + const resultOfMultiple = progress.results.filter( + ({ name }) => name == "multiple" + ); + assertEquals(resultOfMultiple.length, 1); + assert(!!resultOfMultiple[0].measuredRunsMs); + assert(!isNaN(resultOfMultiple[0].measuredRunsAvgMs)); + assertEquals(resultOfMultiple[0].measuredRunsMs.length, 2); + + // The last progress should equal the final result from promise except the state property + progress = progressCallbacks[pc++]; + assertEquals(progress.state, ProgressState.BenchmarkingEnd); + delete progress.state; + assertEquals(progress, benchingResults); + }, +}); + +function dummyBench(name: string, runs = 1): void { + bench({ + name, + runs, + func(b) { + b.start(); + b.stop(); + }, + }); +} diff --git a/std/testing/diff.ts b/std/testing/diff.ts index 97baa089f43e21..da1e827ac04545 100644 --- a/std/testing/diff.ts +++ b/std/testing/diff.ts @@ -1,4 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +/** This module is browser compatible. */ + interface FarthestPoint { y: number; id: number; diff --git a/std/textproto/test.ts b/std/textproto/test.ts index ec66bfd8c2d1c4..7539e977992a73 100644 --- a/std/textproto/test.ts +++ b/std/textproto/test.ts @@ -5,12 +5,12 @@ import { BufReader } from "../io/bufio.ts"; import { TextProtoReader } from "./mod.ts"; -import { stringsReader } from "../io/util.ts"; +import { StringReader } from "../io/readers.ts"; import { assert, assertEquals, assertThrows } from "../testing/asserts.ts"; const { test } = Deno; function reader(s: string): TextProtoReader { - return new TextProtoReader(new BufReader(stringsReader(s))); + return new TextProtoReader(new BufReader(new StringReader(s))); } test({ @@ -187,7 +187,7 @@ test({ const input = "abcdefghijklmnopqrstuvwxyz"; const bufSize = 25; const tp = new TextProtoReader( - new BufReader(stringsReader(input), bufSize) + new BufReader(new StringReader(input), bufSize) ); const line = await tp.readLine(); assertEquals(line, input); diff --git a/std/uuid/README.md b/std/uuid/README.md index fc2a10e9050f51..2b7c7956545135 100644 --- a/std/uuid/README.md +++ b/std/uuid/README.md @@ -11,5 +11,5 @@ import { v4 } from "https://deno.land/std/uuid/mod.ts"; const myUUID = v4.generate(); // Validate a v4 uuid -const isValid = v4.validate(aString); +const isValid = v4.validate(myUUID); ``` diff --git a/std/uuid/tests/isNil.ts b/std/uuid/tests/isNil.ts index 4514576daa7606..1f0db416e2bfac 100644 --- a/std/uuid/tests/isNil.ts +++ b/std/uuid/tests/isNil.ts @@ -1,7 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. import { assert } from "../../testing/asserts.ts"; const { test } = Deno; -// @ts-ignore import { NIL_UUID, isNil } from "../mod.ts"; test({ diff --git a/std/uuid/v5.ts b/std/uuid/v5.ts index 3c04873feca1fd..f982d9745acd24 100644 --- a/std/uuid/v5.ts +++ b/std/uuid/v5.ts @@ -8,7 +8,7 @@ import { } from "./_common.ts"; import { Sha1 } from "../hash/sha1.ts"; import { isString } from "../node/util.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; diff --git a/std/version.ts b/std/version.ts index e05359510d31b6..2199cdfed252e4 100644 --- a/std/version.ts +++ b/std/version.ts @@ -4,4 +4,4 @@ * the cli's API is stable. In the future when std becomes stable, likely we * will match versions with cli as we have in the past. */ -export const VERSION = "0.53.0"; +export const VERSION = "0.56.0"; diff --git a/std/ws/README.md b/std/ws/README.md index c166a5fb6b581a..0c3b7a2a27e6dd 100644 --- a/std/ws/README.md +++ b/std/ws/README.md @@ -7,58 +7,62 @@ ws module is made to provide helpers to create WebSocket client/server. ### Server ```ts -import { serve } from "https://deno.land/std/http/server.ts"; +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +import { serve } from "../../http/server.ts"; import { acceptWebSocket, isWebSocketCloseEvent, isWebSocketPingEvent, -} from "https://deno.land/std/ws/mod.ts"; - -/** websocket echo server */ -const port = Deno.args[0] || "8080"; -console.log(`websocket server is running on :${port}`); -for await (const req of serve(`:${port}`)) { - const { conn, r: bufReader, w: bufWriter, headers } = req; + WebSocket, +} from "../../ws/mod.ts"; +async function handleWs(sock: WebSocket) { + console.log("socket connected!"); try { - const sock = await acceptWebSocket({ + for await (const ev of sock) { + if (typeof ev === "string") { + // text message + console.log("ws:Text", ev); + await sock.send(ev); + } else if (ev instanceof Uint8Array) { + // binary message + console.log("ws:Binary", ev); + } else if (isWebSocketPingEvent(ev)) { + const [, body] = ev; + // ping + console.log("ws:Ping", body); + } else if (isWebSocketCloseEvent(ev)) { + // close + const { code, reason } = ev; + console.log("ws:Close", code, reason); + } + } + } catch (err) { + console.error(`failed to receive frame: ${err}`); + + if (!sock.isClosed) { + await sock.close(1000).catch(console.error); + } + } +} + +if (import.meta.main) { + /** websocket echo server */ + const port = Deno.args[0] || "8080"; + console.log(`websocket server is running on :${port}`); + for await (const req of serve(`:${port}`)) { + const { conn, r: bufReader, w: bufWriter, headers } = req; + acceptWebSocket({ conn, bufReader, bufWriter, headers, - }); - - console.log("socket connected!"); - - try { - for await (const ev of sock) { - if (typeof ev === "string") { - // text message - console.log("ws:Text", ev); - await sock.send(ev); - } else if (ev instanceof Uint8Array) { - // binary message - console.log("ws:Binary", ev); - } else if (isWebSocketPingEvent(ev)) { - const [, body] = ev; - // ping - console.log("ws:Ping", body); - } else if (isWebSocketCloseEvent(ev)) { - // close - const { code, reason } = ev; - console.log("ws:Close", code, reason); - } - } - } catch (err) { - console.error(`failed to receive frame: ${err}`); - - if (!sock.isClosed) { - await sock.close(1000).catch(console.error); - } - } - } catch (err) { - console.error(`failed to accept websocket: ${err}`); - await req.respond({ status: 400 }); + }) + .then(handleWs) + .catch(async (e) => { + console.error(`failed to accept websocket: ${err}`); + await req.respond({ status: 400 }); + }); } } ``` diff --git a/std/ws/example_server.ts b/std/ws/example_server.ts index 3a981595716f08..0d0b2860793b89 100644 --- a/std/ws/example_server.ts +++ b/std/ws/example_server.ts @@ -4,54 +4,55 @@ import { acceptWebSocket, isWebSocketCloseEvent, isWebSocketPingEvent, + WebSocket, } from "./mod.ts"; +async function handleWs(sock: WebSocket): Promise { + console.log("socket connected!"); + try { + for await (const ev of sock) { + if (typeof ev === "string") { + // text message + console.log("ws:Text", ev); + await sock.send(ev); + } else if (ev instanceof Uint8Array) { + // binary message + console.log("ws:Binary", ev); + } else if (isWebSocketPingEvent(ev)) { + const [, body] = ev; + // ping + console.log("ws:Ping", body); + } else if (isWebSocketCloseEvent(ev)) { + // close + const { code, reason } = ev; + console.log("ws:Close", code, reason); + } + } + } catch (err) { + console.error(`failed to receive frame: ${err}`); + + if (!sock.isClosed) { + await sock.close(1000).catch(console.error); + } + } +} + if (import.meta.main) { /** websocket echo server */ const port = Deno.args[0] || "8080"; console.log(`websocket server is running on :${port}`); for await (const req of serve(`:${port}`)) { const { conn, r: bufReader, w: bufWriter, headers } = req; - - try { - const sock = await acceptWebSocket({ - conn, - bufReader, - bufWriter, - headers, + acceptWebSocket({ + conn, + bufReader, + bufWriter, + headers, + }) + .then(handleWs) + .catch(async (e) => { + console.error(`failed to accept websocket: ${e}`); + await req.respond({ status: 400 }); }); - - console.log("socket connected!"); - - try { - for await (const ev of sock) { - if (typeof ev === "string") { - // text message - console.log("ws:Text", ev); - await sock.send(ev); - } else if (ev instanceof Uint8Array) { - // binary message - console.log("ws:Binary", ev); - } else if (isWebSocketPingEvent(ev)) { - const [, body] = ev; - // ping - console.log("ws:Ping", body); - } else if (isWebSocketCloseEvent(ev)) { - // close - const { code, reason } = ev; - console.log("ws:Close", code, reason); - } - } - } catch (err) { - console.error(`failed to receive frame: ${err}`); - - if (!sock.isClosed) { - await sock.close(1000).catch(console.error); - } - } - } catch (err) { - console.error(`failed to accept websocket: ${err}`); - await req.respond({ status: 400 }); - } } } diff --git a/std/ws/example_test.ts b/std/ws/example_test.ts new file mode 100644 index 00000000000000..7e68dc38b4b839 --- /dev/null +++ b/std/ws/example_test.ts @@ -0,0 +1,2 @@ +import "./example_client.ts"; +import "./example_server.ts"; diff --git a/std/ws/mod.ts b/std/ws/mod.ts index 324588af01db5d..f58828aea892f9 100644 --- a/std/ws/mod.ts +++ b/std/ws/mod.ts @@ -8,7 +8,7 @@ import { Sha1 } from "../hash/sha1.ts"; import { writeResponse } from "../http/_io.ts"; import { TextProtoReader } from "../textproto/mod.ts"; import { Deferred, deferred } from "../async/deferred.ts"; -import { assert } from "../testing/asserts.ts"; +import { assert } from "../_util/assert.ts"; import { concat } from "../bytes/mod.ts"; import Conn = Deno.Conn; import Writer = Deno.Writer; @@ -491,7 +491,7 @@ export async function handshake( throw new Error("ws: invalid status line: " + statusLine); } - // @ts-ignore + assert(m.groups); const { version, statusCode } = m.groups; if (version !== "HTTP/1.1" || statusCode !== "101") { throw new Error( diff --git a/std/ws/test.ts b/std/ws/test.ts index a1c396b186c52c..9ef6ff94b363ec 100644 --- a/std/ws/test.ts +++ b/std/ws/test.ts @@ -283,8 +283,8 @@ function dummyConn(r: Reader, w: Writer): Conn { return { rid: -1, closeWrite: (): void => {}, - read: (x): Promise => r.read(x), - write: (x): Promise => w.write(x), + read: (x: Uint8Array): Promise => r.read(x), + write: (x: Uint8Array): Promise => w.write(x), close: (): void => {}, localAddr: { transport: "tcp", hostname: "0.0.0.0", port: 0 }, remoteAddr: { transport: "tcp", hostname: "0.0.0.0", port: 0 }, diff --git a/test_plugin/Cargo.toml b/test_plugin/Cargo.toml index 704cee8314fc4c..3787579a8f1309 100644 --- a/test_plugin/Cargo.toml +++ b/test_plugin/Cargo.toml @@ -9,5 +9,5 @@ publish = false crate-type = ["cdylib"] [dependencies] -futures = "0.3.4" +futures = "0.3.5" deno_core = { path = "../core" } diff --git a/test_plugin/src/lib.rs b/test_plugin/src/lib.rs index 37868b31026580..781bc42594fda3 100644 --- a/test_plugin/src/lib.rs +++ b/test_plugin/src/lib.rs @@ -13,15 +13,16 @@ pub fn deno_plugin_init(interface: &mut dyn Interface) { fn op_test_sync( _interface: &mut dyn Interface, data: &[u8], - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> Op { - if let Some(buf) = zero_copy { - let data_str = std::str::from_utf8(&data[..]).unwrap(); + let data_str = std::str::from_utf8(&data[..]).unwrap(); + let zero_copy = zero_copy.to_vec(); + if !zero_copy.is_empty() { + println!("Hello from plugin. data: {}", data_str); + } + for (idx, buf) in zero_copy.iter().enumerate() { let buf_str = std::str::from_utf8(&buf[..]).unwrap(); - println!( - "Hello from plugin. data: {} | zero_copy: {}", - data_str, buf_str - ); + println!("zero_copy[{}]: {}", idx, buf_str); } let result = b"test"; let result_box: Buf = Box::new(*result); @@ -31,16 +32,17 @@ fn op_test_sync( fn op_test_async( _interface: &mut dyn Interface, data: &[u8], - zero_copy: Option, + zero_copy: &mut [ZeroCopyBuf], ) -> Op { - let data_str = std::str::from_utf8(&data[..]).unwrap().to_string(); + let zero_copy = zero_copy.to_vec(); + if !zero_copy.is_empty() { + let data_str = std::str::from_utf8(&data[..]).unwrap().to_string(); + println!("Hello from plugin. data: {}", data_str); + } let fut = async move { - if let Some(buf) = zero_copy { + for (idx, buf) in zero_copy.iter().enumerate() { let buf_str = std::str::from_utf8(&buf[..]).unwrap(); - println!( - "Hello from plugin. data: {} | zero_copy: {}", - data_str, buf_str - ); + println!("zero_copy[{}]: {}", idx, buf_str); } let (tx, rx) = futures::channel::oneshot::channel::>(); std::thread::spawn(move || { diff --git a/test_plugin/tests/integration_tests.rs b/test_plugin/tests/integration_tests.rs index 17002fc0142b47..8716048b1244a6 100644 --- a/test_plugin/tests/integration_tests.rs +++ b/test_plugin/tests/integration_tests.rs @@ -57,7 +57,7 @@ fn basic() { println!("stderr {}", stderr); } assert!(output.status.success()); - let expected = "Hello from plugin. data: test | zero_copy: test\nPlugin Sync Response: test\nHello from plugin. data: test | zero_copy: test\nPlugin Async Response: test\n"; + let expected = "Hello from plugin. data: test\nzero_copy[0]: test\nzero_copy[1]: 123\nzero_copy[2]: cba\nPlugin Sync Response: test\nHello from plugin. data: test\nzero_copy[0]: test\nzero_copy[1]: 123\nPlugin Async Response: test\n"; assert_eq!(stdout, expected); assert_eq!(stderr, ""); } diff --git a/test_plugin/tests/test.js b/test_plugin/tests/test.js index 8d6146902d5509..fbe58aeb8138af 100644 --- a/test_plugin/tests/test.js +++ b/test_plugin/tests/test.js @@ -33,7 +33,9 @@ function runTestSync() { const response = Deno.core.dispatch( testSync, new Uint8Array([116, 101, 115, 116]), - new Uint8Array([116, 101, 115, 116]) + new Uint8Array([116, 101, 115, 116]), + new Uint8Array([49, 50, 51]), + new Uint8Array([99, 98, 97]) ); console.log(`Plugin Sync Response: ${textDecoder.decode(response)}`); @@ -47,7 +49,8 @@ function runTestAsync() { const response = Deno.core.dispatch( testAsync, new Uint8Array([116, 101, 115, 116]), - new Uint8Array([116, 101, 115, 116]) + new Uint8Array([116, 101, 115, 116]), + new Uint8Array([49, 50, 51]) ); if (response != null || response != undefined) { @@ -80,9 +83,11 @@ function runTestPluginClose() { const preStr = JSON.stringify(resourcesPre, null, 2); const postStr = JSON.stringify(resourcesPost, null, 2); if (preStr !== postStr) { - throw new Error(`Difference in open resources before openPlugin and after Plugin.close(): + throw new Error( + `Difference in open resources before openPlugin and after Plugin.close(): Before: ${preStr} -After: ${postStr}`); +After: ${postStr}` + ); } } diff --git a/tools/format.py b/tools/format.py index ffeaa250dbdb81..f81d99164435d8 100755 --- a/tools/format.py +++ b/tools/format.py @@ -58,11 +58,14 @@ def prettier(): "bin-prettier.js") source_files = get_sources(root_path, ["*.js", "*.json", "*.ts", "*.md"]) if source_files: - print_command("prettier", source_files) - run(["node", script, "--write", "--loglevel=error", "--"] + - source_files, - shell=False, - quiet=True) + max_command_length = 24000 + while len(source_files) > 0: + command = ["node", script, "--write", "--loglevel=error", "--"] + while len(source_files) > 0: + command.append(source_files.pop()) + if len(" ".join(command)) > max_command_length: + run(command, shell=False, quiet=True) + break def yapf(): diff --git a/tools/http_server.py b/tools/http_server.py index 346b319f801377..d143f0ba8e2c48 100755 --- a/tools/http_server.py +++ b/tools/http_server.py @@ -194,8 +194,9 @@ def do_GET(self): def do_POST(self): # Simple echo server for request reflection if "echo_server" in self.path: + status = int(self.headers.getheader('x-status', "200")) self.protocol_version = 'HTTP/1.1' - self.send_response(200, 'OK') + self.send_response(status, 'OK') if self.headers.has_key('content-type'): self.send_header('content-type', self.headers.getheader('content-type')) @@ -206,6 +207,25 @@ def do_POST(self): data_string = self.rfile.read(int(self.headers['Content-Length'])) self.wfile.write(bytes(data_string)) return + if "echo_multipart_file" in self.path: + self.protocol_version = 'HTTP/1.1' + self.send_response(200, 'OK') + self.send_header('Content-type', + 'multipart/form-data;boundary=boundary') + self.end_headers() + file_content = self.rfile.read(int(self.headers['Content-Length'])) + self.wfile.write( + bytes('--boundary\t \r\n' + 'Content-Disposition: form-data; name="field_1"\r\n' + '\r\n' + 'value_1 \r\n' + '\r\n--boundary\r\n' + 'Content-Disposition: form-data; name="file"; ' + 'filename="file.bin"\r\n' + 'Content-Type: application/octet-stream\r\n' + '\r\n') + bytes(file_content) + + bytes('\r\n--boundary--\r\n')) + return self.protocol_version = 'HTTP/1.1' self.send_response(501) self.send_header('content-type', 'text/plain') @@ -363,7 +383,7 @@ def start(s): def spawn(): servers = (server(), redirect_server(), another_redirect_server(), double_redirects_server(), https_server(), - absolute_redirect_server()) + absolute_redirect_server(), inf_redirects_server()) # In order to wait for each of the servers to be ready, we try connecting to # them with a tcp socket. for running_server in servers: diff --git a/tools/hyper_hello/Cargo.toml b/tools/hyper_hello/Cargo.toml index 9029815ccb6956..55a8be7b10d1ee 100644 --- a/tools/hyper_hello/Cargo.toml +++ b/tools/hyper_hello/Cargo.toml @@ -4,8 +4,8 @@ version = "0.0.1" edition = "2018" [dependencies] -hyper = "0.13.5" -tokio = { version = "0.2.20", features = ["full"] } +hyper = "0.13.6" +tokio = { version = "0.2.21", features = ["full"] } [[bin]] name = "hyper_hello"