Slash and burn: build complexity begone!

.github/workflows/docker-build.yml

@@ -10,31 +10,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  calculate_branches:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out current branch
-        uses: actions/checkout@v3.0.2
-        with:
-          fetch-depth: 0  # All history for all tags and branches, since branch calculation step needs that
-
-      - name: Calculate branches to build
-        run: |
-          echo "${GITHUB_REF}"
-          ./calculate-branches.sh --no-fw8 | grep FW9Branch >> "${GITHUB_ENV}"
-
-      - name: Save branch names in output
-        id: branches_to_build
-        run: |
-          echo "FW9Branch=${FW9Branch}"
-          echo "::set-output name=FW9Branch::${FW9Branch}"
-
-    outputs:
-      FW9Branch: ${{ steps.branches_to_build.outputs.FW9Branch }}
-
   build:
     runs-on: ubuntu-latest
-    needs: calculate_branches
 
     # As of 2022-08-16, we build LfMerge for LCM DB version 72 only (and will expand this to include any future DbVersions)
     strategy:
@@ -48,13 +25,6 @@ jobs:
         with:
           fetch-depth: 0  # All history for all tags and branches, since version number script needs that
 
-      - name: Verify current branch
-        run: |
-          git branch --contains HEAD --format '%(refname)'
-          echo FW9 was "${FW9Branch}"
-        env:
-          FW9Branch: ${{ needs.calculate_branches.outputs.FW9Branch }}
-
       - name: Calculate version number
         id: version
         env:
@@ -108,9 +78,6 @@ jobs:
           InformationalVersion: ${{ steps.version.outputs.InformationalVersion }}
         run: docker run --mount type=bind,source="$(pwd)",target=/home/builder/repo --env "BUILD_NUMBER=${BUILD_NUMBER}" --env "DebPackageVersion=${DebPackageVersion}" --env "Version=${MsBuildVersion}" --env "MajorMinorPatch=${MajorMinorPatch}" --env "AssemblyVersion=${AssemblySemVer}" --env "FileVersion=${AssemblySemFileVer}" --env "InformationalVersion=${InformationalVersion}" --name tmp-lfmerge-build-${{matrix.dbversion}} lfmerge-build-${{matrix.dbversion}}
 
-      - name: Collect tarball images
-        run: docker container cp tmp-lfmerge-build-${{matrix.dbversion}}:/home/builder/packages/lfmerge/tarball ./
-
       - name: Compress tarball images for faster uploads
         run: time (tar cf - tarball | gzip -c9 > tarball.tar.gz)
 
@@ -121,12 +88,10 @@ jobs:
     outputs:
       MsBuildVersion: ${{ steps.output_version_number.outputs.VersionFor7000072 }}
       TagFor7000072: ${{ steps.output_version_number.outputs.TagFor7000072 }}
-      FW9Branch: ${{ needs.calculate_branches.outputs.FW9Branch }}
 
   release:
     needs: build
     uses: ./.github/workflows/release.yml
     with:
       MsBuildVersion: ${{ needs.build.outputs.MsBuildVersion }}
       TagFor7000072: ${{ needs.build.outputs.TagFor7000072 }}
-      FW9Branch: ${{ needs.build.outputs.FW9Branch }}

.github/workflows/release.yml

@@ -6,9 +6,6 @@ on:
       MsBuildVersion:
         required: true
         type: string
-      FW9Branch:
-        required: true
-        type: string
       TagFor7000072:
         required: true
         type: string
@@ -18,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3.0.2
       with:
         fetch-depth: 0
 
@@ -31,12 +28,11 @@ jobs:
     - name: Tag release branches
       if: github.event_name == 'push' && github.ref == 'refs/heads/live'
       env:
-        FW9Branch: ${{ inputs.FW9Branch }}
         TAG72: ${{ inputs.TagFor7000072 }}
       run: |
         git config --global user.name "github-actions"
         git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
-        git tag -f -a -m "Release ${TAG72}" "${TAG72}" "refs/remotes/origin/${FW9Branch}"
+        git tag -f -a -m "Release ${TAG72}" "${TAG72}" "refs/remotes/origin/live"
         git push -v origin "${TAG72}"
 
     - name: Download build artifacts
@@ -54,16 +50,6 @@ jobs:
     - name: Verify that uncompress step worked
       run: ls -lR tarball
 
-    - name: Restore executable bits
-      # GitHub artifacts system strips executable bits, so restore them here
-      # TODO: Now that we upload .tar.gz format, no longer needed since tar format preserves Unix permissions. Verify, then remove this section
-      run: |
-        chmod +x tarball/lfmerge*/usr/bin/lfmerge
-        chmod +x tarball/lfmerge*/usr/bin/lfmergeqm
-        chmod +x tarball/lfmerge*/usr/lib/lfmerge/*/chorusmerge
-        chmod +x tarball/lfmerge*/usr/lib/lfmerge/*/startlfmerge
-        chmod +x tarball/lfmerge*/usr/lib/lfmerge/*/Mercurial/hg
-
     - name: Login to GHCR
       if: github.event_name == 'push' && github.ref == 'refs/heads/live'
       uses: docker/login-action@v1

.vscode/launch.json

@@ -34,31 +34,10 @@
 			}
 		},
 		*/
-		{
-			"name": "Launch unit tests",
-			"env": {
-				// Display on main display :0, or on a Xephyr :2.
-				"DISPLAY": ":0",
-				// When launching LfMerge, use this mono.
-				"PATH": "/opt/mono5-sil/bin:${env:PATH}",
-				// Path to the rest of the necessary enviromnent settings for LfMerge. This is handled by a mono wrapper script in /opt.
-				"MONO_ENVIRON": "${workspaceRoot}/environ"
-			},
-			"type": "mono",
-			"request": "launch",
-			"program": "${workspaceRoot}/packages/NUnit.Runners.Net4.2.6.4/tools/nunit.exe",
-			"args": [
-				"LfMerge.Core.Tests.dll"
-			],
-			"cwd": "${workspaceRoot}/output/Debug",
-			"console": "integratedTerminal"
-		},
-		{
-			"name": "Attach",
-			"type": "mono",
-			"request": "attach",
-			"address": "localhost",
-			"port": 55555
-		}
-	]
+    {
+      "name": ".NET Core Attach",
+      "type": "coreclr",
+      "request": "attach",
+    },
+  ]
 }

.vscode/tasks.json

@@ -0,0 +1,47 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "build",
+            "command": "dotnet",
+            "type": "shell",
+            "args": [
+                "build",
+                // Ask dotnet build to generate full paths for file names.
+                "/property:GenerateFullPaths=true",
+                // Do not generate summary otherwise it leads to duplicate errors in Problems panel
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "group": "build",
+            "presentation": {
+                "reveal": "silent"
+            },
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": ".NET Core Test with debugger",
+            "type": "process",
+            "isBackground": true,
+            "command": "dotnet",
+            "args": [ "test" ],
+            "options":
+                {
+                    "cwd": "${workspaceFolder}",
+                    "env":
+                    {
+                        "VSTEST_HOST_DEBUG": "1"
+                    },
+                },
+            "group": "test", "presentation":
+                {
+                    "echo": true,
+                    "reveal": "always",
+                    "focus": false,
+                    "panel": "shared"
+                },
+            "problemMatcher": []
+        },
+    ]
+}

Dockerfile

@@ -9,51 +9,35 @@ RUN test -n "$BUILDER_UID"
 ENV BUILDER_UID="$BUILDER_UID"
 
 # # Build as a non-root user
-RUN useradd -u "${BUILDER_UID:-DEFAULT_BUILDER_UID}" -d /home/builder -g users -G www-data,fieldworks,systemd-journal -m -s /bin/bash builder ; \
+RUN useradd -u "${BUILDER_UID:-DEFAULT_BUILDER_UID}" -d /home/builder -g users -G www-data,fieldworks -m -s /bin/bash builder ; \
     echo "builder ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers; \
 	chown -R builder:users /build
 
-# Any setup unique to the various builds goes in one of these four images
-FROM lfmerge-builder-base AS lfmerge-build-7000068
-ENV DbVersion=7000068
-ENV DBVERSIONPATH=/usr/lib/lfmerge/7000068
-ENV RUN_UNIT_TESTS=0
-ENV UPDATE_ASSEMBLYINFO_BY_SCRIPT=1
-ENV NUNIT_VERSION_MAJOR=2
-# To run specific unit tests, set TEST_SPEC env var, e.g.:
-# ENV TEST_SPEC=LfMerge.Core.Tests.Actions.SynchronizeActionTests.SynchronizeAction_CustomReferenceAtomicField_DoesNotThrowExceptionDuringSync
-
-FROM lfmerge-builder-base AS lfmerge-build-7000069
-ENV DbVersion=7000069
-ENV DBVERSIONPATH=/usr/lib/lfmerge/7000069
-ENV RUN_UNIT_TESTS=0
-ENV UPDATE_ASSEMBLYINFO_BY_SCRIPT=1
-ENV NUNIT_VERSION_MAJOR=2
-# ENV TEST_SPEC=LfMerge.Core.Tests.Actions.SynchronizeActionTests.SynchronizeAction_CustomReferenceAtomicField_DoesNotThrowExceptionDuringSync
-
-FROM lfmerge-builder-base AS lfmerge-build-7000070
-ENV DbVersion=7000070
-ENV DBVERSIONPATH=/usr/lib/lfmerge/7000070
-ENV RUN_UNIT_TESTS=0
-ENV UPDATE_ASSEMBLYINFO_BY_SCRIPT=1
-ENV NUNIT_VERSION_MAJOR=2
-# ENV TEST_SPEC=LfMerge.Core.Tests.Actions.SynchronizeActionTests.SynchronizeAction_CustomReferenceAtomicField_DoesNotThrowExceptionDuringSync
+# Any setup unique to one specific DbVersion build goes in one of the images below
 
 FROM lfmerge-builder-base AS lfmerge-build-7000072
 ENV DbVersion=7000072
 ENV DBVERSIONPATH=/usr/lib/lfmerge/7000072
 ENV RUN_UNIT_TESTS=0
-ENV UPDATE_ASSEMBLYINFO_BY_SCRIPT=0
 ENV NUNIT_VERSION_MAJOR=3
 # ENV TEST_SPEC=LfMerge.Core.Tests.Actions.SynchronizeActionTests.SynchronizeAction_CustomReferenceAtomicField_DoesNotThrowExceptionDuringSync
 
 FROM lfmerge-build-${DbVersion} AS lfmerge-build
 
+# So unit tests can run
+RUN mkdir -p /var/lib/languageforge/lexicon/sendreceive/ \
+	&& cd /var/lib/languageforge/lexicon/sendreceive/ && mkdir Templates state editqueue syncqueue webwork && cd - \
+	&& chown -R builder:users /var/lib/languageforge/lexicon/sendreceive
+
 USER builder
+WORKDIR /home/builder/repo
+# Git repo should be mounted under /home/builder/repo when run
+# E.g., `docker run --mount type=bind,source="$(pwd)",target=/home/builder/repo
+
+# NuGet package cache dir should be bind-mounted from home of running user
+# E.g., `docker run --mount type=bind,source="${HOME}/.nuget/packages",target=/home/builder/.nuget/packages
+RUN mkdir -p /home/builder/.nuget/packages
 
-# Git repo should be mounted under ${HOME}/packages/lfmerge when run
-# E.g., `docker run --mount type=bind,source="$(pwd)",target=/home/builder/packages/lfmerge`
-RUN mkdir -p /home/builder/repo /home/builder/packages/lfmerge
-CMD /home/builder/repo/docker/scripts/build-and-test.sh ${DbVersion}
+CMD docker/scripts/build-and-test.sh ${DbVersion}
 # CMD doesn't actually run the script, it just gives `docker run` a default.
 # So it's okay for the Git repo to not be mounted yet.

Dockerfile.builder-base

@@ -5,16 +5,9 @@ WORKDIR /build/lfmerge
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DOTNET_CLI_TELEMETRY_OPTOUT=1
-RUN apt-get update && apt-get install -y gnupg
 
-COPY sil-packages-key.gpg .
-COPY sil-packages-testing-key.gpg .
-RUN apt-key add sil-packages-key.gpg
-RUN apt-key add sil-packages-testing-key.gpg
-RUN echo 'deb http://linux.lsdev.sil.org/ubuntu bionic main' > /etc/apt/sources.list.d/llso-experimental.list
-RUN echo 'deb http://linux.lsdev.sil.org/ubuntu bionic-experimental main' >> /etc/apt/sources.list.d/llso-experimental.list
 # Dependencies from Debian "control" file
-RUN apt-get update && apt-get install -y sudo debhelper devscripts cli-common-dev iputils-ping cpp python-dev pkg-config mono5-sil mono5-sil-msbuild libicu-dev lfmerge-fdo && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y curl sudo iputils-ping cpp python-is-python2 python2-dev pkg-config libicu-dev && rm -rf /var/lib/apt/lists/*
 
 # Ensure fieldworks group exists in case lfmerge-fdo package didn't install it, and that www-data is part of that group
 # Also ensure that www-data has a .local dir in its home directory (/var/www) since some of lfmerge's dependencies assume that $HOME/.local exists

Dockerfile.finalresult

@@ -6,27 +6,19 @@ FROM ghcr.io/sillsdev/lfmerge-base:runtime
 # install LFMerge prerequisites
 # tini - PID 1 handler
 # python - required by Mercurial, which is bundled in the LFMerge installation
-# rsyslog - lfmerge logs to rsyslog and expects this to exist
-# rsyslog customizations (imklog reads kernel messages, which isn't allowed or desired in Docker containers)
-# logrotate - ensure logs won't grow without bound, and that old logs get cleaned up automatically
 # iputils-ping - Chorus (part of LFMerge) requires the "ping" command to be available on the command line
 # less - so we can read syslog during manual debugging of issues
 # vim-tiny - so we can edit state files (to change HOLD to IDLE) during manual debugging of issues
-RUN curl -L http://linux.lsdev.sil.org/downloads/sil-testing.gpg | apt-key add - \
-&& echo "deb http://linux.lsdev.sil.org/ubuntu bionic main" > /etc/apt/sources.list.d/linux-lsdev-sil-org.list \
-&& apt-get update \
-&& apt-get install --yes --no-install-recommends tini python rsyslog logrotate iputils-ping inotify-tools less vim-tiny \
-&& rm -rf /var/lib/apt/lists/* \
-&& sed -i '/load="imklog"/s/^/#/' /etc/rsyslog.conf
+RUN apt-get update \
+	&& apt-get install --yes --no-install-recommends tini python iputils-ping inotify-tools less vim-tiny \
+	&& rm -rf /var/lib/apt/lists/*
 
 ADD tarball/lfmerge* /
 
 RUN mkdir -m 02775 -p /var/lib/languageforge/lexicon/sendreceive/syncqueue /var/lib/languageforge/lexicon/sendreceive/webwork /var/lib/languageforge/lexicon/sendreceive/Templates /var/lib/languageforge/lexicon/sendreceive/state && chown -R www-data:www-data /var/lib/languageforge
 
 RUN install -d -o www-data -g www-data -m 02775 /var/run/lfmerge
 
-# TODO: Turn this into environment variables, because we want to be able to just set env vars in k8s config and restart the container
-COPY lfmerge.conf /etc/languageforge/conf/sendreceive.conf
 COPY sudoers.d.lfmerge.conf /etc/sudoers.d/lfmerge
 COPY mercurial-cacerts.rc /etc/mercurial/hgrc
 COPY lfmergeqm-background.sh /

Dockerfile.runtime-base

@@ -5,16 +5,9 @@ WORKDIR /build/lfmerge
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DOTNET_CLI_TELEMETRY_OPTOUT=1
-RUN apt-get update && apt-get install -y gnupg
 
-COPY sil-packages-key.gpg .
-COPY sil-packages-testing-key.gpg .
-RUN apt-key add sil-packages-key.gpg
-RUN apt-key add sil-packages-testing-key.gpg
-RUN echo 'deb http://linux.lsdev.sil.org/ubuntu bionic main' > /etc/apt/sources.list.d/llso-experimental.list
-RUN echo 'deb http://linux.lsdev.sil.org/ubuntu bionic-experimental main' >> /etc/apt/sources.list.d/llso-experimental.list
 # Dependencies from Debian "control" file
-RUN apt-get update && apt-get install -y sudo debhelper devscripts cli-common-dev iputils-ping cpp python-dev pkg-config mono5-sil mono5-sil-msbuild libicu-dev lfmerge-fdo && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y curl sudo iputils-ping cpp python-is-python2 python2-dev pkg-config libicu-dev && rm -rf /var/lib/apt/lists/*
 
 # Ensure fieldworks group exists in case lfmerge-fdo package didn't install it, and that www-data is part of that group
 # Also ensure that www-data has a .local dir in its home directory (/var/www) since some of lfmerge's dependencies assume that $HOME/.local exists