fix(linter/plugins): handle utf16 characters within comment spans (#14768)

Part of #14564. Corrects start and end offsets to accommodate two byte
characters. Conversion of UTF-8 indices to UTF-16 takes place on the
rust side for performance.

Author: lilnasy

apps/oxlint/test/e2e.test.ts

@@ -220,4 +220,8 @@ describe('oxlint CLI', () => {
   it('should support comments-related APIs in `context.sourceCode`', async () => {
     await testFixture('comments');
   });
+
+  it('should support UTF16 characters in source code and comments with correct spans', async () => {
+    await testFixture('unicode-comments');
+  });
 });

apps/oxlint/test/fixtures/unicode-comments/.oxlintrc.json

@@ -0,0 +1,7 @@
+{
+  "jsPlugins": ["./plugin.ts"],
+  "categories": { "correctness": "off" },
+  "rules": {
+    "unicode-comments/unicode-comments": "error"
+  }
+}

apps/oxlint/test/fixtures/unicode-comments/files/unicode-comments.js

@@ -0,0 +1,30 @@
+// Unicode test with emojis and multi-byte characters
+const greeting = 'Hello 🌍'; // Line comment with emoji
+
+/**
+ * Function with emoji in JSDoc
+ * @param {string} name - User's name 👤
+ * @returns {string} Greeting message
+ */
+function greetUser(name) {
+  // Comment with multiple emojis 🚀⭐💫
+  const message = `Hello ${name}! 🌟`;
+  /* Block comment with unicode: ñáéíóú */
+  return message;
+}
+
+/* Multi-byte comment: 你好世界 */
+const 你好世界 = 'Testing üöä'; // Line comment: ñáéíóú
+
+/**
+ * JSDoc with emojis and unicode: 你好 👋
+ * @param {number} count - Number of items 🔢
+ */
+function processItems(count) {
+  // Comment with mixed unicode: αβγδε русский עברית
+  const result = count * 2; /* Block: ñáéíóú 🚀 */
+  return result;
+}
+
+// Final comment with emoji: 🎉✨🎊
+const finalVar = 'Done ✅';

apps/oxlint/test/fixtures/unicode-comments/output.snap.md

@@ -0,0 +1,93 @@
+# Exit code
+1
+
+# stdout
+```
+  x unicode-comments(unicode-comments): getAllComments: [
+  |     {
+  |         "type": "Line",
+  |         "value": " Unicode test with emojis and multi-byte characters"
+  |     },
+  |     {
+  |         "type": "Line",
+  |         "value": " Line comment with emoji"
+  |     },
+  |     {
+  |         "type": "Block",
+  |         "value": "*\n * Function with emoji in JSDoc\n * @param {string} name - User's name 👤\n * @returns {string} Greeting message\n "
+  |     },
+  |     {
+  |         "type": "Line",
+  |         "value": " Comment with multiple emojis 🚀⭐💫"
+  |     },
+  |     {
+  |         "type": "Block",
+  |         "value": " Block comment with unicode: ñáéíóú "
+  |     },
+  |     {
+  |         "type": "Block",
+  |         "value": " Multi-byte comment: 你好世界 "
+  |     },
+  |     {
+  |         "type": "Line",
+  |         "value": " Line comment: ñáéíóú"
+  |     },
+  |     {
+  |         "type": "Block",
+  |         "value": "*\n * JSDoc with emojis and unicode: 你好 👋\n * @param {number} count - Number of items 🔢\n "
+  |     },
+  |     {
+  |         "type": "Line",
+  |         "value": " Comment with mixed unicode: αβγδε русский עברית"
+  |     },
+  |     {
+  |         "type": "Block",
+  |         "value": " Block: ñáéíóú 🚀 "
+  |     },
+  |     {
+  |         "type": "Line",
+  |         "value": " Final comment with emoji: 🎉✨🎊"
+  |     }
+  | ]
+    ,-[files/unicode-comments.js:2:1]
+  1 |     // Unicode test with emojis and multi-byte characters
+  2 | ,-> const greeting = 'Hello 🌍'; // Line comment with emoji
+  3 | |   
+  4 | |   /**
+  5 | |    * Function with emoji in JSDoc
+  6 | |    * @param {string} name - User's name 👤
+  7 | |    * @returns {string} Greeting message
+  8 | |    */
+  9 | |   function greetUser(name) {
+ 10 | |     // Comment with multiple emojis 🚀⭐💫
+ 11 | |     const message = `Hello ${name}! 🌟`;
+ 12 | |     /* Block comment with unicode: ñáéíóú */
+ 13 | |     return message;
+ 14 | |   }
+ 15 | |   
+ 16 | |   /* Multi-byte comment: 你好世界 */
+ 17 | |   const 你好世界 = 'Testing üöä'; // Line comment: ñáéíóú
+ 18 | |   
+ 19 | |   /**
+ 20 | |    * JSDoc with emojis and unicode: 你好 👋
+ 21 | |    * @param {number} count - Number of items 🔢
+ 22 | |    */
+ 23 | |   function processItems(count) {
+ 24 | |     // Comment with mixed unicode: αβγδε русский עברית
+ 25 | |     const result = count * 2; /* Block: ñáéíóú 🚀 */
+ 26 | |     return result;
+ 27 | |   }
+ 28 | |   
+ 29 | |   // Final comment with emoji: 🎉✨🎊
+ 30 | `-> const finalVar = 'Done ✅';
+    `----
+
+Found 0 warnings and 1 error.
+Finished in Xms on 1 file using X threads.
+```
+
+# stderr
+```
+WARNING: JS plugins are experimental and not subject to semver.
+Breaking changes are possible while JS plugins support is under development.
+```

apps/oxlint/test/fixtures/unicode-comments/plugin.ts

@@ -0,0 +1,28 @@
+import type { Plugin, Rule } from '../../../dist/index.js';
+
+const unicodeCommentsRule: Rule = {
+  create(context) {
+    const { sourceCode } = context;
+    const { ast } = sourceCode;
+
+    context.report({
+      message: `getAllComments: ${
+        JSON.stringify(sourceCode.getAllComments().map(c => ({ type: c.type, value: c.value })), null, 4)
+      }`,
+      node: ast,
+    });
+
+    return {};
+  },
+};
+
+const plugin: Plugin = {
+  meta: {
+    name: 'unicode-comments',
+  },
+  rules: {
+    'unicode-comments': unicodeCommentsRule,
+  },
+};
+
+export default plugin;

crates/oxc_linter/src/lib.rs

@@ -433,6 +433,7 @@ impl Linter {
             // Convert spans to UTF-16
             let span_converter = Utf8ToUtf16::new(program.source_text);
             span_converter.convert_program(program);
+            span_converter.convert_comments(&mut program.comments);
 
             // Get offset of `Program` within buffer (bottom 32 bits of pointer)
             let program_offset = ptr::from_ref(program) as u32;