Add compile_flags.txt for clangd

.gitignore

@@ -11,8 +11,7 @@ node_modules
 /npm/workerd/install.js
 /npm/workerd/lib/
 
-### Added by Hedron's Bazel Compile Commands Extractor: https://github.com/hedronvision/bazel-compile-commands-extractor
-# The external link: Differs on Windows vs macOS/Linux, so we can't check it in. The pattern needs to not have a trailing / because it's a symlink on macOS/Linux.
+# The external link for compile_flags.txt: Differs on Windows vs macOS/Linux, so we can't check it in. The pattern needs to not have a trailing / because it's a symlink on macOS/Linux.
 /external
 # Bazel output symlinks: Same reasoning as /external. You need the * because people can change the name of the directory your repository is cloned into, changing the bazel-<workspace_name> symlink.
 /bazel-*

.vscode/tasks.json

@@ -204,11 +204,11 @@
       }
     },
     {
-      "label": "Generate compile_commands.json",
-      "command": "bazel",
-      "args": ["run", "//:refresh_compile_commands"],
-      "options": {
-        "cwd": "${workspaceFolder}"
+      // Create ${workspaceFolder}/external for clangd on opening VSCode.
+      "label": "Symlink external directory",
+      "command": "${workspaceFolder}/tools/unix/create-external.sh",
+      "windows": {
+        "command": "${workspaceFolder}/tools/windows/create-external.bat",
       },
       "group": "build",
       "problemMatcher": [],
@@ -217,7 +217,8 @@
         "panel": "dedicated"
       },
       "runOptions": {
-        "runOn": "default"
+        "reevaluateOnRerun": false,
+        "runOn": "folderOpen"
       }
     }
   ]

BUILD.bazel

@@ -1,5 +1,4 @@
 load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
-load("@hedron_compile_commands//:refresh_compile_commands.bzl", "refresh_compile_commands")
 load("@aspect_rules_js//npm:defs.bzl", "npm_link_package", "npm_package")
 load("@npm//:defs.bzl", "npm_link_all_packages")
 load("@npm//:capnpc-ts/package_json.bzl", capnpc_ts_bin = "bin")
@@ -15,10 +14,6 @@ cc_capnp_library(
     defines = [ "WORKERD_ICU_DATA_EMBED" ],
 )
 
-refresh_compile_commands(
-    name = "refresh_compile_commands",
-)
-
 npm_link_all_packages(name = "node_modules")
 
 npm_link_package(

WORKSPACE

@@ -350,19 +350,3 @@ new_local_repository(
         visibility = ["//visibility:public"])""",
     path = "empty",
 )
-
-# ========================================================================================
-# Tools
-
-# Hedron's Compile Commands Extractor for Bazel
-# https://github.com/hedronvision/bazel-compile-commands-extractor
-http_archive(
-    name = "hedron_compile_commands",
-    sha256 = "ab6c6b4ceaf12b224e571ec075fd79086c52c3430993140bb2ed585b08dfc552",
-    strip_prefix = "bazel-compile-commands-extractor-d1e95ec162e050b04d0a191826f9bc478de639f7",
-    url = "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/d1e95ec162e050b04d0a191826f9bc478de639f7.tar.gz",
-)
-
-load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
-
-hedron_compile_commands_setup()

compile_flags.txt

@@ -0,0 +1,97 @@
+-std=c++20
+-stdlib=libc++
+-xc++
+-nostdinc
+-Ibazel-bin
+-Ibazel-bin/external/capnp-cpp/src
+-Ibazel-bin/external/com_googlesource_chromium_base_trace_event_common/_virtual_includes/trace_event_common
+-Iexternal/capnp-cpp/src
+-Isrc
+-isystem/usr/include
+-isystem/usr/include/x86_64-linux-gnu
+-isystem/usr/lib/llvm-15/include/c++/v1
+-isystem/usr/lib/llvm-15/lib/clang/15.0.7/include
+-isystem/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include
+-isystem/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/c++/v1
+-isystemC:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.35.32215\include
+-isystemc:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt
+-isystembazel-bin/external/v8
+-isystembazel-bin/external/v8/icu
+-isystembazel-bin/src
+-isystemexternal/brotli/c/include
+-isystemexternal/com_googlesource_chromium_icu/source/common
+-isystemexternal/icu
+-isystemexternal/ssl/src/include
+-isystemexternal/v8
+-isystemexternal/v8/include
+-isystemexternal/zlib
+-D_FORTIFY_SOURCE=1
+-DBAZEL_CURRENT_REPOSITORY=\"\"
+-DCAPNP_VERSION=11000
+-DDEBUG
+-DGOOGLE3
+-DHAVE_DLOPEN=0
+-DKJ_HAS_LIBDL
+-DKJ_HAS_OPENSSL
+-DKJ_HAS_ZLIB
+-DKJ_SAVE_ACQUIRED_LOCK_INFO=0
+-DKJ_TRACK_LOCK_BLOCKING=0
+-DSQLITE_ENABLE_FTS5
+-DSQLITE_ENABLE_NORMALIZE
+-DSQLITE_MAX_ALLOCATION_SIZE=16777216
+-DSQLITE_PRINTF_PRECISION_LIMIT=100000
+-DUCONFIG_ONLY_HTML_CONVERSION=1
+-DUNISTR_FROM_CHAR_EXPLICIT=
+-DUNISTR_FROM_STRING_EXPLICIT=
+-DUSE_CHROMIUM_ICU=1
+-DU_CHARSET_IS_UTF8=1
+-DU_COMMON_IMPLEMENTATION
+-DU_ENABLE_DYLOAD=0
+-DU_ENABLE_RESOURCE_TRACING=0
+-DU_ENABLE_TRACING=1
+-DU_I18N_IMPLEMENTATION
+-DU_ICUDATAENTRY_IN_COMMON
+-DU_USING_ICU_NAMESPACE=0
+-DV8_31BIT_SMIS_ON_64BIT_ARCH
+-DV8_ADVANCED_BIGINT_ALGORITHMS
+-DV8_COMPRESS_POINTERS
+-DV8_COMPRESS_POINTERS_IN_SHARED_CAGE
+-DV8_CONCURRENT_MARKING
+-DV8_DEPRECATION_WARNINGS
+-DV8_ENABLE_CHECKS
+-DV8_ENABLE_LAZY_SOURCE_POSITIONS
+-DV8_ENABLE_TURBOFAN
+-DV8_ENABLE_WEBASSEMBLY
+-DV8_HAVE_TARGET_OS
+-DV8_IMMINENT_DEPRECATION_WARNINGS
+-DV8_SHORT_BUILTIN_CALLS
+-DV8_TARGET_ARCH_X64
+-DV8_TARGET_OS_LINUX
+-DV8_TYPED_ARRAY_MAX_SIZE_IN_HEAP=64
+-DWORKERD_ICU_DATA_EMBED
+-Wall
+-Werror=dangling
+-Werror=implicit-fallthrough
+-Werror=return-stack-address
+-Werror=return-type
+-Werror=switch
+-Werror=uninitialized
+-Werror=unreachable-code
+-Werror=unused-function
+-Werror=unused-lambda-capture
+-Werror=unused-variable
+-Wextra
+-Wno-builtin-macro-redefined
+-Wno-free-nonheap-object
+-Wno-ignored-qualifiers
+-Wno-missing-field-initializers
+-Wno-sign-compare
+-Wno-strict-aliasing
+-Wno-unused-parameter
+-Wself-assign
+-Wthread-safety
+-Wunused-but-set-parameter
+-Wunused-function
+-Wunused-lambda-capture
+-Wunused-variable
+-no-canonical-prefixes

docs/vscode.md

@@ -34,7 +34,6 @@ The [.vscode/tasks.json](../.vscode/tasks.json) file provides a few useful tasks
 * Bazel run all tests (dbg)
 * Bazel run all tests (fastbuild)
 * Bazel run all tests (opt)
-* Generate compile_commands.json
 * Generate rust-project.json
 
 The keyboard shortcut for **Tasks: Run Build Task** is `shift+ctrl+b` on Linux and Windows, `shift+cmd+b` on OS X.
@@ -68,26 +67,29 @@ Launching "workerd test case (dbg)" will prompt for a test binary to debug, the
 
 Launching "workerd wd-test case (dbg)" will prompt for wd-test file to provide to workerd to debug, the default is `src/workerd/api/node/path-test.wd-test`.
 
-## Generating compile_commands.json
+## Clangd code completion, navigation, language server
 
-We use clangd for code completion and navigation with the editor. Clangd requires a `compile_commands.json` file to find and process the project source code. This can be generated from within VSCode using the **Run Build Task** command and choosing "Generate compile_commands.json" (or via `shift+ctrl+b` on Linux / Windows, `shift_cmd+b` on OS X).
+We use clangd for code completion and navigation within the Visual Code. We use the simple
+[compile_flags.txt](../compile_flags.txt) option provide compiler arguments for clangd to analyze sources.
 
-Alternatively, you can generate `compile_commands.json` at the command-line directly:
-
-```sh
-bazel run //:refresh_compile_commands
+If `compile_flags.txt` is not working well on your system, try running:
 ```
+bazel build --copt="-MD" --cxxopt="-MD" //src/workerd/server:workerd
+```
+to generate dependency files and:
+```
+find bazel-out/ -name '*.d'`
+```
+to locate the generated dependency files. These files will help you align the include paths in
+`compile_flags.txt` with the ones that the bazel build is using.
 
-There is an issue between workerd's bazel setup and Hedron's compile_commands.json generator (tracked in
-https://github.com/cloudflare/workerd/issues/506).
-
-If you hit any problems, you may find this helps (though it takes a long time!):
+There is also a script [clangd-check.sh](../tools/unix/clangd-check.sh) that checks every `.h` and
+`.c++` file in the workerd source tree. Fixing errors there inevitably improves the experience in
+Visual Studio Code.
 
-```sh
-bazel clean --expunge
-bazel test //...
-bazel run //:refresh_compile_commands
-```
+In the past we used [Hedron's Bazel Compile Commands Extractor](https://github.com/hedronvision/bazel-compile-commands-extractor)
+to generate a `compile_commands.json` file for clangd, but this was slow and unreliable for the `workerd` use case
+(see https://github.com/cloudflare/workerd/issues/506).
 
 ## Miscellaneous tips
 

src/workerd/jsg/function.h

@@ -7,9 +7,9 @@
 //
 // Handles wrapping a C++ function so that it can be called from JavaScript, and vice versa.
 
-#include "wrappable.h"
-#include "resource.h"  // for ArgumentIndexes
 #include <kj/function.h>
+#include "resource.h"  // for ArgumentIndexes
+#include "wrappable.h"
 
 namespace workerd::jsg {
 

src/workerd/jsg/iterator.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include "jsg.h"
+#include "struct.h"
 #include <concepts>
 #include <deque>
 

src/workerd/jsg/promise.h

@@ -4,11 +4,10 @@
 
 #pragma once
 
+#include <kj/async.h>
+#include "jsg.h"
 #include "util.h"
 #include "wrappable.h"
-#include "jsg.h"
-#include "web-idl.h"
-#include <kj/async.h>
 
 namespace workerd::jsg {
 

src/workerd/jsg/resource.h

@@ -10,13 +10,13 @@
 // can call back to the class's methods. This differs from, say, a struct type, which will be deeply
 // converted into a JS object when passed into JS.
 
-#include "util.h"
-#include "wrappable.h"
-#include "jsg.h"
 #include <kj/tuple.h>
 #include <kj/debug.h>
 #include <type_traits>
 #include <kj/map.h>
+#include "util.h"
+#include "wrappable.h"
+#include "jsg.h"
 #include <typeindex>
 
 namespace std {

src/workerd/jsg/rtti.h

@@ -9,9 +9,10 @@
 // Can be used to generate typescript type, dynamically invoke methods, fuzz, check backward
 // compatibility etc.
 
+#include <kj/map.h>
+#include <capnp/message.h>
 #include <workerd/jsg/jsg.h>
 #include <workerd/jsg/rtti.capnp.h>
-#include <kj/map.h>
 
 namespace workerd::jsg::rtti {
 

src/workerd/jsg/value.h

@@ -10,7 +10,6 @@
 
 #include "util.h"
 #include "wrappable.h"
-#include "jsg.h"
 #include "web-idl.h"
 #include <kj/time.h>
 #include <kj/debug.h>

src/workerd/jsg/web-idl.h

@@ -15,7 +15,7 @@
 namespace workerd::jsg {
 class UsvString;
 class UsvStringPtr;
-}  // namespace workerd::cjfs
+}  // namespace workerd::jsg
 
 namespace workerd::jsg::webidl {
 

tools/unix/clangd-check.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# A script to scans workerd sources with clangd to look for issues
+# that make affect language services.
+
+output_dir=$(mktemp -d)
+top_of_tree=$(git rev-parse --show-toplevel)
+
+cd "${top_of_tree}"
+for i in $(find . -name '*.h' -o -name '*.c++'); do
+    j=${output_dir}/$(echo $i | sed  -e 's@^\./@@' -e s@/@_@g)
+    echo Scanning $i =\> $j
+    clangd --check=$i 1>$j 2>&1
+done

tools/unix/create-external.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Create a symlink to Bazel's external/ directory. This simplifies the
+# paths provided to clangd in compile_flags.txt for language server
+# use.
+
+output_path=$(bazel info output_path)
+workspace=$(bazel info workspace)
+external="${workspace}/external"
+ln -sfF "${output_path}/../../../external" "${external}"
+
+# Temporary warning that compile_commands.json exists and will
+# interfere with the intended clangd setup.
+compile_commands="${workspace}/compile_commands.json"
+if [ -f "compile_commands.json" ] ; then
+  cat<<COMPILE_COMMANDS_WARNING
+WARNING: This workspace has a compile_commands.json file, but workerd
+has moved to using compile_flags.txt for clangd instead. To improve
+code completion and navigation in your editor, consider running:
+
+  rm "${compile_commands}"
+
+COMPILE_COMMANDS_WARNING
+fi

tools/windows/create-external.bat

@@ -0,0 +1,22 @@
+@ECHO off
+SETLOCAL EnableDelayedExpansion
+
+@REM cmd.exe script to ensure external/ directory is present for clangd.
+
+FOR /f %%i IN ('bazel info output_path') do SET "output_path=%%i"
+FOR /f %%i IN ('bazel info workspace') do SET "workspace=%%i"
+SET "external=%output_path%\..\..\..\external"
+
+IF NOT EXIST "%workspace%\external" (
+  MKLINK /J "%workspace%\external" "%external%"
+)
+
+SET "compile_commands=%workspace%\compile_commands.json"
+IF EXIST "%compile_commands%" (
+  ECHO.WARNING: This workspace has a compile_commands.json file, but workerd
+  ECHO.has moved to using compile_flags.txt for clangd instead. To improve
+  ECHO.code completion and navigation in your editor, consider running:
+  ECHO.
+  ECHO.  DEL /q "%compile_commands%"
+  ECHO.
+)