Add CI for MSVC using dkml-workflows

Implements #6535

.gitattributes

@@ -2,6 +2,7 @@
 *.rst             text eol=lf
 *.c               text eol=lf
 *.t               text eol=lf -linguist-detectable
+*.ps1             text working-tree-encoding=UTF-16 eol=crlf
 dune              text eol=lf
 dune.inc          text eol=lf
 .gitignore        text eol=lf

.github/workflows/workflow.yml

@@ -102,6 +102,36 @@ jobs:
         run: opam install ./dune-configurator.opam
         if: ${{ matrix.configurator == true }}
 
+  dkml:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - gh_os: windows-2019
+            abi_pattern: win32-windows_x86
+            dkml_host_abi: windows_x86
+          - gh_os: windows-2019
+            abi_pattern: win32-windows_x86_64
+            dkml_host_abi: windows_x86_64
+    runs-on: ${{ matrix.gh_os }}
+    name: MSVC / ${{ matrix.dkml_host_abi }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup DKML on a Windows host
+        if: startsWith(matrix.dkml_host_abi, 'windows_')
+        uses: ./ci/setup-dkml/gh-windows/pre
+
+      - name: Build and test the package on Windows host
+        if: startsWith(matrix.dkml_host_abi, 'windows_')
+        shell: msys2 {0}
+        run: ci/build-test.sh
+
+      - name: Teardown DKML on a Windows host
+        if: startsWith(matrix.dkml_host_abi, 'windows_')
+        uses: ./ci/setup-dkml/gh-windows/post
+
   nix:
     name: Nix
     strategy:

.gitignore

@@ -24,3 +24,7 @@ result
 
 .DS_Store
 nix/profiles/
+
+# dkml desktop CI
+/msys64
+/.ci

CHANGES.md

@@ -22,6 +22,8 @@ Unreleased
 - Remove "Entering Directory" messages for `$ dune install`. (#6513,
   @rgrinberg)
 
+- Add CI testing with MSVC (todo, @jonahbeckford)
+
 3.6.0 (2022-11-14)
 ------------------
 

Makefile

@@ -32,6 +32,7 @@ ctypes \
 # Dependencies recommended for developing dune locally,
 # but not wanted in CI
 DEV_DEPS := \
+"dkml-workflows>=1.1.0" \
 patdiff
 
 TEST_OCAMLVERSION := 4.14.0
@@ -79,6 +80,10 @@ melange:
 dev-deps: melange
 	opam install -y $(TEST_DEPS)
 
+.PHONE: dev-deps-sans-melange
+dev-deps-sans-melange:
+	opam install -y $(TEST_DEPS)
+
 .PHONY: dev-switch
 dev-switch:
 	opam update
@@ -106,6 +111,9 @@ test-melange: $(BIN)
 test-all: $(BIN)
 	$(BIN) build @runtest @runtest-js @runtest-coq @runtest-melange
 
+test-all-sans-melange: $(BIN)
+	$(BIN) build @runtest @runtest-js @runtest-coq
+
 .PHONY: check
 check: $(BIN)
 	@$(BIN) build @check
@@ -142,6 +150,37 @@ livedoc:
 update-jbuilds: $(BIN)
 	$(BIN) build @doc/runtest --auto-promote
 
+# you will need to use a dev-switch or do opam install dkml-workflows.
+# we run --auto-promote twice since first may fail (ex. missing files with Unable to resolve symlink)
+update-dkml: $(BIN)
+	opam exec -- generate-setup-dkml-scaffold
+	$(BIN) build @gen-dkml --auto-promote || $(BIN) build @gen-dkml --auto-promote
+	$(BIN) build @ci/fmt --auto-promote || $(BIN) build @ci/fmt --auto-promote
+
+# assumes MSYS2 or Cygwin, and Visual Studio. Do not use 'with-dkml make ...'
+desktop-ci-windows_x86:
+	if command -v pwsh; then \
+	  pwsh ./ci/setup-dkml/pc/setup-dkml-windows_x86.ps1; \
+	else \
+	  powershell ./ci/setup-dkml/pc/setup-dkml-windows_x86.ps1; \
+	fi
+	/usr/bin/env PATH=/usr/bin \
+	  dkml_host_abi=windows_x86 abi_pattern=win32-windows_x86_64; \
+	  opam_root=.ci/o exe_ext=.exe \
+	  /bin/sh ci/build-test.sh
+
+# assumes MSYS2 or Cygwin, and Visual Studio. Do not use 'with-dkml make ...'
+desktop-ci-windows_x86_64:
+	if command -v pwsh; then \
+	  pwsh ./ci/setup-dkml/pc/setup-dkml-windows_x86_64.ps1; \
+	else \
+	  powershell ./ci/setup-dkml/pc/setup-dkml-windows_x86_64.ps1; \
+	fi
+	/usr/bin/env PATH=/usr/bin \
+	  dkml_host_abi=windows_x86 abi_pattern=win32-windows_x86_64 \
+	  opam_root=.ci/o exe_ext=.exe \
+	  /bin/sh ci/build-test.sh
+
 # If the first argument is "run"...
 ifeq (dune,$(firstword $(MAKECMDGOALS)))
   # use the rest as arguments for "run"

ci/build-test.sh

@@ -0,0 +1,130 @@
+#!/bin/sh
+set -euf
+
+# shellcheck disable=SC2154
+echo "
+=============
+build-test.sh
+=============
+.
+------
+Matrix
+------
+dkml_host_abi=$dkml_host_abi
+abi_pattern=$abi_pattern
+opam_root=$opam_root
+exe_ext=${exe_ext:-}
+.
+"
+
+# Set project directory
+if [ -n "${CI_PROJECT_DIR:-}" ]; then
+    PROJECT_DIR="$CI_PROJECT_DIR"
+elif [ -n "${PC_PROJECT_DIR:-}" ]; then
+    PROJECT_DIR="$PC_PROJECT_DIR"
+elif [ -n "${GITHUB_WORKSPACE:-}" ]; then
+    PROJECT_DIR="$GITHUB_WORKSPACE"
+else
+    PROJECT_DIR="$PWD"
+fi
+if [ -x /usr/bin/cygpath ]; then
+    PROJECT_DIR=$(/usr/bin/cygpath -au "$PROJECT_DIR")
+fi
+
+# PATH. Add opamrun
+export PATH="$PROJECT_DIR/.ci/sd4/opamrun:$PATH"
+
+# Initial Diagnostics (optional but useful)
+opamrun switch
+opamrun list
+opamrun var
+opamrun config report
+opamrun option
+opamrun exec -- ocamlc -config
+
+# Update
+opamrun update
+
+# ------------------
+# Dune's build logic.
+#   For now it mimics the "build" job in .github/workflows/workflow.yml
+# ------------------
+
+echo ======== Unpin Dune from DKML defaults
+opamrun pin list
+opamrun pin remove dune --no-action
+opamrun pin remove dune-configurator --no-action
+
+echo ======== Set git user
+git config --global user.name 'github-actions[bot]'
+git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+
+echo ======== Install ocamlfind-secondary and ocaml-secondary-compiler, if needed
+opamrun install ./dune.opam --deps-only --with-test
+
+case "$dkml_host_abi" in
+darwin_*)
+    echo ======== Install system deps on macOS
+    brew install coreutils
+    ;;
+esac
+
+# dune doesn't have any additional dependencies so we can build it right
+# away this makes it possible to see build errors as soon as possible
+echo ======== Build boot dune.exe
+opamrun exec -- make _boot/dune.exe
+
+echo ======== Ensure Dune can build itself
+opamrun exec -- make bootstrap
+
+case "$dkml_host_abi" in
+windows_*)
+    # Mitigate bug in opam:
+    #
+    #    #=== ERROR while compiling dune.3.6.0 =========================================#
+    #    # path        Z:\source\dune
+    #    # command     C:\Users\beckf\AppData\Local\Programs\DiskuvOCaml\tools\MSYS2\usr\bin\cp.exe -PRp Z:\source\dune\.ci\o\dkml\.opam-switch\sources\dune.3.6.0 Z:\source\dune\.ci\o\dkml\.opam-switch\build\dune.3.6.0
+    #    # exit-code   1
+    #    # env-file    Z:\source\dune\.ci\o\log\log-685364-345b91.env
+    #    # output-file Z:\source\dune\.ci\o\log\log-685364-345b91.out
+    #    ### output ###
+    #    # /usr/bin/cp: cannot create regular file 'Z:\source\dune\.ci\o\dkml\.opam-switch\build\dune.3.6.0/dune': File exists
+    #
+    # Basically the 'cp' fallback logic in opam's src/core/opamSystem.ml:copy_dir is broken.
+    # Using 'rsync' avoids that, so install it on MSYS2.
+    if command -v pacman; then
+        echo ======== Install rsync on Win32
+        pacman --sync --needed --noconfirm rsync
+    fi
+
+    echo ======== Install deps on Win32
+    opamrun install ./dune-configurator.opam --deps-only --with-test --yes
+
+    echo ======== Run test suite on Win32
+    opamrun exec -- make test-windows
+    ;;
+*)
+    echo ======== Install deps on Unix
+    opamrun install . --deps-only --with-test --yes
+
+    # Remove DKML pins (keep alphabetical; the pins will eventually be upstreamed in DKML)
+    opamrun pin remove base --no-action
+    opamrun pin remove core_kernel --no-action
+    opamrun pin remove ocamlformat --no-action
+    opamrun pin remove ppx_expect --no-action
+    opamrun pin remove time_now --no-action
+
+    # This should have been:
+    #   opam exec -- make dev-deps
+    # but melange requires ocaml 4.14 (DKML hasn't been updated as of 2022-11)
+    opamrun exec -- make dev-deps-sans-melange
+
+    echo ======== Run test suite on Unix
+    # This should have been:
+    #   opam exec -- make test
+    # but melange requires ocaml 4.14 (DKML hasn't been updated as of 2022-11)
+    opamrun exec -- make test-all-sans-melange
+esac
+
+echo ======== Build configurator
+opamrun install ./dune-configurator.opam --yes

ci/setup-dkml/gh-darwin/post/action.yml

@@ -0,0 +1,73 @@
+# setup-dkml
+#   Short form: sd4
+
+name: post-dkml
+author: Diskuv, Inc.
+description: Teardown after building OCaml native executables for Darwin
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Full matrix variables
+      shell: bash
+      # Every matrix variable lookup in this Action should use the output of this step. Even
+      # the matrix variables that the user must specify (ex. dkml_host_abi) should be
+      # referenced using [steps.full_matrix_vars.outputs.dkml_host_abi] rather than
+      # [matrix.dkml_host_abi] so that there is a single place to edit for variable changes.
+      id: full_matrix_vars
+      run: |
+        # Select correct Darwin matrix variables
+        case "${{ matrix.dkml_host_abi }}" in
+        
+        darwin_x86_64)
+          dkml_host_os='darwin';
+          opam_root_cacheable='/Users/runner/.opam';
+          abi_pattern='macos-darwin_all';
+          gh_os='macos-latest';
+          gh_unix_shell='sh';
+          bootstrap_opam_version='2.2.0-dkml20220801T155940Z';
+          dkml_host_abi='darwin_x86_64';
+          opam_root='/Users/runner/.opam' ;;
+          
+        *)  echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Darwin action.yml"; exit 107 ;;
+        esac
+
+        add() {
+          echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV
+        }
+
+        add dkml_host_abi "$dkml_host_abi"
+        add abi_pattern "$abi_pattern"
+        add opam_root "$opam_root"
+        add opam_root_cacheable "$opam_root_cacheable"
+        add exe_ext "${exe_ext:-}"
+        add bootstrap_opam_version "${bootstrap_opam_version:-}"
+        add ocaml_options "${ocaml_options:-}"
+
+    - name: Teardown DKML build apparatus
+      shell: bash
+      env:
+        _STUB_FOR_AUTOGEN: "ON" # 
+
+        # autogen from global_env_vars.
+        DEFAULT_DKML_COMPILER: '4.12.1-v1.0.2'
+        PIN_BASE: 'v0.14.3'
+        PIN_BIGSTRINGAF: '0.8.0'
+        PIN_CORE_KERNEL: 'v0.14.2'
+        PIN_CTYPES_FOREIGN: '0.19.2-windowssupport-r4'
+        PIN_CTYPES: '0.19.2-windowssupport-r4'
+        PIN_CURLY: '0.2.1-windows-env_r2'
+        PIN_DIGESTIF: '1.0.1'
+        PIN_DUNE: '2.9.3+shim.1.0.2~r0'
+        PIN_DUNE_CONFIGURATOR: '2.9.3'
+        PIN_DKML_APPS: '1.0.1'
+        PIN_OCAMLBUILD: '0.14.0'
+        PIN_OCAMLFIND: '1.9.1'
+        PIN_OCP_INDENT: '1.8.2-windowssupport'
+        PIN_PPX_EXPECT: 'v0.14.1'
+        PIN_PTIME: '0.8.6-msvcsupport'
+        PIN_TIME_NOW: 'v0.14.0'
+        PIN_WITH_DKML: '1.0.1'
+      run: |
+        sh .ci/sd4/run-teardown-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE"

ci/setup-dkml/gh-darwin/post/dune

@@ -0,0 +1,17 @@
+; DO NOT EDIT THIS FILE. It is auto-generated by generate-setup-dkml-scaffold
+; Typical upgrade steps:
+;   opam upgrade dkml-workflows && opam exec -- generate-setup-dkml-scaffold && dune build '@gen-dkml' --auto-promote
+
+(rule
+ (alias gen-dkml)
+ (target action.gen.yml)
+ (action
+  (setenv
+   OCAMLRUNPARAM
+   b
+   (run gh-dkml-action-yml --phase post --output-darwin %{target}))))
+
+(rule
+ (alias gen-dkml)
+ (action
+  (diff action.yml action.gen.yml)))