From c7468f77396bdc0f5cf20769ca425640e1ec036d Mon Sep 17 00:00:00 2001
From: Balavishnu <balavishnu@HME778.local>
Date: Sun, 4 Jun 2023 22:08:33 +0530
Subject: [PATCH] adds implementation details

---
 designs/2023-sharding/README.md | 62 ++++++++++++++++++++++++++++-----
 1 file changed, 53 insertions(+), 9 deletions(-)

diff --git a/designs/2023-sharding/README.md b/designs/2023-sharding/README.md
index ed6049c1..f248e9a5 100644
--- a/designs/2023-sharding/README.md
+++ b/designs/2023-sharding/README.md
@@ -20,9 +20,9 @@ The implementation of sharding in the ESLint package involves the following step
 
 - **Configuration**: Introduce a new configuration option or flag in the ESLint configuration file to enable sharding. This option can specify the number of shards or the desired granularity of the shards.
 
-    ```
-    eslint --shard=1/3
-    ```
+  ```
+  eslint --shard=1/3
+  ```
 
 - **Workload Division**: Analyze the input codebase and divide it into smaller units of work called shards. The division can be based on files, directories, or any other logical grouping that allows for parallel processing. Each shard should be self-contained and independent of the others. For example, larger files can be distributed across shards to balance the workload. We could also expose api to get custom division statergy.
 
@@ -32,6 +32,54 @@ The implementation of sharding in the ESLint package involves the following step
 
 - **Error Handling**: Implement appropriate error handling mechanisms to handle failures in individual shards. If a shard encounters an error during linting, it should be reported as part of the aggregated results, along with any relevant error information.
 
+## Detailed Design
+
+1. Get the files in a deterministic order.
+   in `lib/cli.js`, we can get all the files. Update the `executeOnFiles` function
+
+   ```js
+   const allFiles = [];
+   for (const { config, filePath, ignored } of fileEnumerator.iterateFiles(
+     patterns
+   )) {
+     /* ... */
+     allFiles.push({ config, filePath, ignored });
+   }
+   ```
+
+2. Divide the files into shards. The number of shards is determined by the `--shard` option.
+
+   ```js
+   const sortedFiles = allFiles.sort((a, b) =>
+     a.filePath.localeCompare(b.filePath)
+   );
+   ```
+
+3. Lint these sorted files
+
+   ```js
+   const shard = parseInt(cliOptions.shard);
+   const totalShards = parseInt(cliOptions.totalShards);
+   const shardFiles = allFiles.filter(
+     (_, index) => index % totalShards === shard - 1
+   );
+
+   shardFiles.forEach(({ filePath, config }) => {
+     const result = verifyText({
+       text: fs.readFileSync(filePath, "utf8"),
+       filePath,
+       config,
+       cwd,
+       fix,
+       allowInlineConfig,
+       reportUnusedDisableDirectives,
+       fileEnumerator,
+       linter,
+     });
+     results.push(result);
+   });
+   ```
+
 ## Documentation
 
 The addition of sharding support should be documented in the ESLint user documentation. The documentation should cover the configuration options, explain the benefits and considerations when using sharding, and provide examples of how to set up and utilize sharding effectively. Additionally, a formal announcement on the ESLint blog can be made to explain the motivation behind introducing sharding and its impact on linting performance.
@@ -48,7 +96,6 @@ Introducing sharding in the ESLint package has several potential drawbacks to co
 
 ## Backwards Compatibility Analysis
 
-
 The introduction of sharding in the ESLint package should not affect existing ESLint users by default. Sharding should be an opt-in feature, enabled through configuration options or flags. Users who do not wish to utilize sharding can continue to use ESLint as before without any disruption.
 
 ## Alternatives
@@ -64,23 +111,20 @@ Manually splitting the project to run in parallel example by @bmish in https://g
     "lint:js:dir1": "eslint --cache dir1/",
     "lint:js:dir2": "eslint --cache dir2/",
     "lint:js:dir3": "eslint --cache dir3/",
-    "lint:js:other": "eslint --cache --ignore-pattern dir1/ --ignore-pattern dir2/ --ignore-pattern dir3/ .",
+    "lint:js:other": "eslint --cache --ignore-pattern dir1/ --ignore-pattern dir2/ --ignore-pattern dir3/ ."
   }
 }
 ```
 
-
 ## Open Questions
 
 - What process/thread management strategy should be employed to handle the shards effectively?
-    
 - How to allow custom division statergy?
 
 ## Help Needed
 
 As a first-time contributor, I would greatly appreciate guidance and assistance from the ESLint core team and the community in implementing the sharding feature. While I have experience as a programmer, contributing to a large-scale project like ESLint involves understanding the codebase, adhering to coding standards, and following the established contribution process.
 
-
 ## Frequently Asked Questions
 
 Q: Will enabling sharding affect the linting results or introduce any inconsistencies?
@@ -93,4 +137,4 @@ A: Sharding should be compatible with ESLint plugins and custom rules. However,
 
 ## Related Discussions
 
-[Change Request: add ability to shard eslint](https://github.com/eslint/eslint/issues/16726)
\ No newline at end of file
+[Change Request: add ability to shard eslint](https://github.com/eslint/eslint/issues/16726)