stage1 is now a hybrid of C++ and Zig

This modifies the build process of Zig to put all of the source files
into libcompiler.a, except main.cpp and userland.cpp.

Next, the build process links main.cpp, userland.cpp, and libcompiler.a
into zig1. userland.cpp is a shim for functions that will later be
replaced with self-hosted implementations.

Next, the build process uses zig1 to build src-self-hosted/stage1.zig
into libuserland.a, which does not depend on any of the things that
are shimmed in userland.cpp, such as translate-c.

Finally, the build process re-links main.cpp and libcompiler.a, except
with libuserland.a instead of userland.cpp. Now the shims are replaced
with .zig code. This provides all of the Zig standard library to the
stage1 C++ compiler, and enables us to move certain things to userland,
such as translate-c.

As a proof of concept I have made the `zig zen` command use text defined
in userland. I added `zig translate-c-2` which is a work-in-progress
reimplementation of translate-c in userland, which currently calls
`std.debug.panic("unimplemented")` and you can see the stack trace makes
it all the way back into the C++ main() function (Thanks LemonBoy for
improving that!).

This could potentially let us move other things into userland, such as
hashing algorithms, the entire cache system, .d file parsing, pretty
much anything that libuserland.a itself doesn't need to depend on.

This can also let us have `zig fmt` in stage1 without the overhead
of child process execution, and without the initial compilation delay
before it gets cached.

See #1964

CMakeLists.txt

@@ -407,6 +407,12 @@ set(SOFTFLOAT_LIBRARIES embedded_softfloat)
 
 find_package(Threads)
 
+# CMake doesn't let us create an empty executable, so we hang on to this one separately.
+set(ZIG_MAIN_SRC "${CMAKE_SOURCE_DIR}/src/main.cpp")
+
+# This is our shim which will be replaced by libuserland written in Zig.
+set(ZIG1_SHIM_SRC "${CMAKE_SOURCE_DIR}/src/userland.cpp")
+
 set(ZIG_SOURCES
     "${CMAKE_SOURCE_DIR}/src/analyze.cpp"
     "${CMAKE_SOURCE_DIR}/src/ast_render.cpp"
@@ -423,7 +429,6 @@ set(ZIG_SOURCES
     "${CMAKE_SOURCE_DIR}/src/ir_print.cpp"
     "${CMAKE_SOURCE_DIR}/src/libc_installation.cpp"
     "${CMAKE_SOURCE_DIR}/src/link.cpp"
-    "${CMAKE_SOURCE_DIR}/src/main.cpp"
     "${CMAKE_SOURCE_DIR}/src/os.cpp"
     "${CMAKE_SOURCE_DIR}/src/parser.cpp"
     "${CMAKE_SOURCE_DIR}/src/range_set.cpp"
@@ -6635,19 +6640,19 @@ add_library(zig_cpp STATIC ${ZIG_CPP_SOURCES})
 set_target_properties(zig_cpp PROPERTIES
     COMPILE_FLAGS ${EXE_CFLAGS}
 )
+install(TARGETS zig_cpp DESTINATION "${ZIG_CPP_LIB_DIR}")
 
 add_library(opt_c_util STATIC ${OPTIMIZED_C_SOURCES})
 set_target_properties(opt_c_util PROPERTIES
     COMPILE_FLAGS "${OPTIMIZED_C_FLAGS}"
 )
 
-add_executable(zig ${ZIG_SOURCES})
-set_target_properties(zig PROPERTIES
+add_library(compiler STATIC ${ZIG_SOURCES})
+set_target_properties(compiler PROPERTIES
     COMPILE_FLAGS ${EXE_CFLAGS}
     LINK_FLAGS ${EXE_LDFLAGS}
 )
-
-target_link_libraries(zig LINK_PUBLIC
+target_link_libraries(compiler LINK_PUBLIC
     zig_cpp
     opt_c_util
     ${SOFTFLOAT_LIBRARIES}
@@ -6656,24 +6661,58 @@ target_link_libraries(zig LINK_PUBLIC
     ${LLVM_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT}
 )
-
 if(NOT MSVC)
-    target_link_libraries(zig LINK_PUBLIC ${LIBXML2})
+    target_link_libraries(compiler LINK_PUBLIC ${LIBXML2})
 endif()
 
 if(MINGW)
-    target_link_libraries(zig LINK_PUBLIC ${Z3_LIBRARIES})
+    target_link_libraries(compiler LINK_PUBLIC ${Z3_LIBRARIES})
 endif()
 
 if(ZIG_DIA_GUIDS_LIB)
-    target_link_libraries(zig LINK_PUBLIC ${ZIG_DIA_GUIDS_LIB})
+    target_link_libraries(compiler LINK_PUBLIC ${ZIG_DIA_GUIDS_LIB})
 endif()
 
 if(MSVC OR MINGW)
-    target_link_libraries(zig LINK_PUBLIC version)
+    target_link_libraries(compiler LINK_PUBLIC version)
+endif()
+
+add_executable(zig1 "${ZIG_MAIN_SRC}" "${ZIG1_SHIM_SRC}")
+set_target_properties(zig1 PROPERTIES
+    COMPILE_FLAGS ${EXE_CFLAGS}
+    LINK_FLAGS ${EXE_LDFLAGS}
+)
+target_link_libraries(zig1 compiler)
+
+if(WIN32)
+    set(LIBUSERLAND "${CMAKE_BINARY_DIR}/userland.lib")
+elseif(APPLE)
+    set(LIBUSERLAND "${CMAKE_BINARY_DIR}/userland.o")
+else()
+    set(LIBUSERLAND "${CMAKE_BINARY_DIR}/libuserland.a")
 endif()
+add_custom_command(
+    OUTPUT "${LIBUSERLAND}"
+    COMMAND zig1 ARGS build
+        --override-std-dir std
+        --override-lib-dir "${CMAKE_SOURCE_DIR}"
+        libuserland
+        "-Doutput-dir=${CMAKE_BINARY_DIR}"
+    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    DEPENDS
+        "${CMAKE_SOURCE_DIR}/src-self-hosted/stage1.zig"
+        "${CMAKE_SOURCE_DIR}/src-self-hosted/translate_c.zig"
+)
+add_custom_target(userland_target DEPENDS "${LIBUSERLAND}")
+add_executable(zig "${ZIG_MAIN_SRC}")
+set_target_properties(zig PROPERTIES
+    COMPILE_FLAGS ${EXE_CFLAGS}
+    LINK_FLAGS ${EXE_LDFLAGS}
+)
+target_link_libraries(zig compiler "${LIBUSERLAND}")
+add_dependencies(zig userland_target)
 install(TARGETS zig DESTINATION bin)
-install(TARGETS zig_cpp DESTINATION "${ZIG_CPP_LIB_DIR}")
+
 
 foreach(file ${ZIG_C_HEADER_FILES})
     get_filename_component(file_dir "${C_HEADERS_DEST}/${file}" DIRECTORY)

build.zig

@@ -65,6 +65,8 @@ pub fn build(b: *Builder) !void {
 
     b.default_step.dependOn(&exe.step);
 
+    addLibUserlandStep(b);
+
     const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse false;
     const skip_release_small = b.option(bool, "skip-release-small", "Main test suite skips release-small builds") orelse skip_release;
     const skip_release_fast = b.option(bool, "skip-release-fast", "Main test suite skips release-fast builds") orelse skip_release;
@@ -380,3 +382,20 @@ const Context = struct {
     dia_guids_lib: []const u8,
     llvm: LibraryDep,
 };
+
+fn addLibUserlandStep(b: *Builder) void {
+    const artifact = if (builtin.os == .macosx)
+        b.addObject("userland", "src-self-hosted/stage1.zig")
+    else
+        b.addStaticLibrary("userland", "src-self-hosted/stage1.zig");
+    artifact.disable_gen_h = true;
+    const libuserland_step = b.step("libuserland", "Build the userland compiler library for use in stage1");
+    libuserland_step.dependOn(&artifact.step);
+
+    const output_dir = b.option(
+        []const u8,
+        "output-dir",
+        "For libuserland step, where to put the output",
+    ) orelse return;
+    artifact.setOutputDir(output_dir);
+}

src-self-hosted/main.zig

@@ -858,23 +858,7 @@ fn cmdHelp(allocator: *Allocator, args: []const []const u8) !void {
     try stdout.write(usage);
 }
 
-const info_zen =
-    \\
-    \\ * Communicate intent precisely.
-    \\ * Edge cases matter.
-    \\ * Favor reading code over writing code.
-    \\ * Only one obvious way to do things.
-    \\ * Runtime crashes are better than bugs.
-    \\ * Compile errors are better than runtime crashes.
-    \\ * Incremental improvements.
-    \\ * Avoid local maximums.
-    \\ * Reduce the amount one must remember.
-    \\ * Minimize energy spent on coding style.
-    \\ * Together we serve end users.
-    \\
-    \\
-;
-
+const info_zen = @import("stage1.zig").info_zen;
 fn cmdZen(allocator: *Allocator, args: []const []const u8) !void {
     try stdout.write(info_zen);
 }

src-self-hosted/stage1.zig

@@ -0,0 +1,27 @@
+// This is Zig code that is used by both stage1 and stage2.
+// The prototypes in src/userland.h must match these definitions.
+comptime {
+    _ = @import("translate_c.zig");
+}
+
+pub const info_zen =
+    \\
+    \\ * Communicate intent precisely.
+    \\ * Edge cases matter.
+    \\ * Favor reading code over writing code.
+    \\ * Only one obvious way to do things.
+    \\ * Runtime crashes are better than bugs.
+    \\ * Compile errors are better than runtime crashes.
+    \\ * Incremental improvements.
+    \\ * Avoid local maximums.
+    \\ * Reduce the amount one must remember.
+    \\ * Minimize energy spent on coding style.
+    \\ * Together we serve end users.
+    \\
+    \\
+;
+
+export fn stage2_zen(ptr: *[*]const u8, len: *usize) void {
+    ptr.* = &info_zen;
+    len.* = info_zen.len;
+}

src-self-hosted/translate_c.zig

@@ -0,0 +1,8 @@
+// This is the userland implementation of translate-c which will be used by both stage1
+// and stage2. Currently it's not used by anything, as it's not feature complete.
+
+const std = @import("std");
+
+export fn stage2_translate_c() void {
+    std.debug.panic("unimplemented");
+}

src/codegen.cpp

@@ -19,6 +19,7 @@
 #include "target.hpp"
 #include "util.hpp"
 #include "zig_llvm.h"
+#include "userland.h"
 
 #include <stdio.h>
 #include <errno.h>
@@ -92,27 +93,32 @@ static const char *symbols_that_llvm_depends_on[] = {
 };
 
 CodeGen *codegen_create(Buf *main_pkg_path, Buf *root_src_path, const ZigTarget *target,
-    OutType out_type, BuildMode build_mode, Buf *zig_lib_dir, Buf *override_std_dir,
+    OutType out_type, BuildMode build_mode, Buf *override_lib_dir, Buf *override_std_dir,
     ZigLibCInstallation *libc, Buf *cache_dir)
 {
     CodeGen *g = allocate<CodeGen>(1);
 
     codegen_add_time_event(g, "Initialize");
 
     g->libc = libc;
-    g->zig_lib_dir = zig_lib_dir;
     g->zig_target = target;
     g->cache_dir = cache_dir;
 
+    if (override_lib_dir == nullptr) {
+        g->zig_lib_dir = get_zig_lib_dir();
+    } else {
+        g->zig_lib_dir = override_lib_dir;
+    }
+
     if (override_std_dir == nullptr) {
         g->zig_std_dir = buf_alloc();
-        os_path_join(zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir);
+        os_path_join(g->zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir);
     } else {
         g->zig_std_dir = override_std_dir;
     }
 
     g->zig_c_headers_dir = buf_alloc();
-    os_path_join(zig_lib_dir, buf_create_from_str("include"), g->zig_c_headers_dir);
+    os_path_join(g->zig_lib_dir, buf_create_from_str("include"), g->zig_c_headers_dir);
 
     g->build_mode = build_mode;
     g->out_type = out_type;
@@ -8147,7 +8153,7 @@ static void detect_libc(CodeGen *g) {
     }
 }
 
-AstNode *codegen_translate_c(CodeGen *g, Buf *full_path) {
+AstNode *codegen_translate_c(CodeGen *g, Buf *full_path, bool use_userland_implementation) {
     Buf *src_basename = buf_alloc();
     Buf *src_dirname = buf_alloc();
     os_path_split(full_path, src_dirname, src_basename);
@@ -8159,6 +8165,12 @@ AstNode *codegen_translate_c(CodeGen *g, Buf *full_path) {
 
     init(g);
 
+    if (use_userland_implementation) {
+        // TODO improve this
+        stage2_translate_c();
+        zig_panic("TODO");
+    }
+
     ZigList<ErrorMsg *> errors = {0};
     AstNode *root_node;
     Error err = parse_h_file(&root_node, &errors, buf_ptr(full_path), g, nullptr);

src/codegen.hpp

@@ -50,7 +50,7 @@ ZigPackage *codegen_create_package(CodeGen *g, const char *root_src_dir, const c
 void codegen_add_assembly(CodeGen *g, Buf *path);
 void codegen_add_object(CodeGen *g, Buf *object_path);
 
-AstNode *codegen_translate_c(CodeGen *g, Buf *path);
+AstNode *codegen_translate_c(CodeGen *g, Buf *path, bool use_userland_implementation);
 
 Buf *codegen_generate_builtin_source(CodeGen *g);
 

src/compiler.cpp

@@ -179,24 +179,24 @@ Buf *get_zig_lib_dir(void) {
     return &saved_lib_dir;
 }
 
-Buf *get_zig_std_dir() {
+Buf *get_zig_std_dir(Buf *zig_lib_dir) {
     if (saved_std_dir.list.length != 0) {
         return &saved_std_dir;
     }
     buf_resize(&saved_std_dir, 0);
 
-    os_path_join(get_zig_lib_dir(), buf_create_from_str("std"), &saved_std_dir);
+    os_path_join(zig_lib_dir, buf_create_from_str("std"), &saved_std_dir);
 
     return &saved_std_dir;
 }
 
-Buf *get_zig_special_dir() {
+Buf *get_zig_special_dir(Buf *zig_lib_dir) {
     if (saved_special_dir.list.length != 0) {
         return &saved_special_dir;
     }
     buf_resize(&saved_special_dir, 0);
 
-    os_path_join(get_zig_std_dir(), buf_sprintf("special"), &saved_special_dir);
+    os_path_join(get_zig_std_dir(zig_lib_dir), buf_sprintf("special"), &saved_special_dir);
 
     return &saved_special_dir;
 }

src/compiler.hpp

@@ -16,7 +16,7 @@ Error get_compiler_id(Buf **result);
 Buf *get_self_dynamic_linker_path(void);
 
 Buf *get_zig_lib_dir(void);
-Buf *get_zig_special_dir(void);
-Buf *get_zig_std_dir(void);
+Buf *get_zig_special_dir(Buf *zig_lib_dir);
+Buf *get_zig_std_dir(Buf *zig_lib_dir);
 
 #endif