diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d8141ac2..5c5a1c04 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -10,31 +10,8 @@ on: workflow_dispatch: jobs: - calculate_branches: - runs-on: ubuntu-latest - steps: - - name: Check out current branch - uses: actions/checkout@v3.0.2 - with: - fetch-depth: 0 # All history for all tags and branches, since branch calculation step needs that - - - name: Calculate branches to build - run: | - echo "${GITHUB_REF}" - ./calculate-branches.sh --no-fw8 | grep FW9Branch >> "${GITHUB_ENV}" - - - name: Save branch names in output - id: branches_to_build - run: | - echo "FW9Branch=${FW9Branch}" - echo "::set-output name=FW9Branch::${FW9Branch}" - - outputs: - FW9Branch: ${{ steps.branches_to_build.outputs.FW9Branch }} - build: runs-on: ubuntu-latest - needs: calculate_branches # As of 2022-08-16, we build LfMerge for LCM DB version 72 only (and will expand this to include any future DbVersions) strategy: @@ -48,13 +25,6 @@ jobs: with: fetch-depth: 0 # All history for all tags and branches, since version number script needs that - - name: Verify current branch - run: | - git branch --contains HEAD --format '%(refname)' - echo FW9 was "${FW9Branch}" - env: - FW9Branch: ${{ needs.calculate_branches.outputs.FW9Branch }} - - name: Calculate version number id: version env: @@ -108,9 +78,6 @@ jobs: InformationalVersion: ${{ steps.version.outputs.InformationalVersion }} run: docker run --mount type=bind,source="$(pwd)",target=/home/builder/repo --env "BUILD_NUMBER=${BUILD_NUMBER}" --env "DebPackageVersion=${DebPackageVersion}" --env "Version=${MsBuildVersion}" --env "MajorMinorPatch=${MajorMinorPatch}" --env "AssemblyVersion=${AssemblySemVer}" --env "FileVersion=${AssemblySemFileVer}" --env "InformationalVersion=${InformationalVersion}" --name tmp-lfmerge-build-${{matrix.dbversion}} lfmerge-build-${{matrix.dbversion}} - - name: Collect tarball images - run: docker container cp tmp-lfmerge-build-${{matrix.dbversion}}:/home/builder/packages/lfmerge/tarball ./ - - name: Compress tarball images for faster uploads run: time (tar cf - tarball | gzip -c9 > tarball.tar.gz) @@ -121,7 +88,6 @@ jobs: outputs: MsBuildVersion: ${{ steps.output_version_number.outputs.VersionFor7000072 }} TagFor7000072: ${{ steps.output_version_number.outputs.TagFor7000072 }} - FW9Branch: ${{ needs.calculate_branches.outputs.FW9Branch }} release: needs: build @@ -129,4 +95,3 @@ jobs: with: MsBuildVersion: ${{ needs.build.outputs.MsBuildVersion }} TagFor7000072: ${{ needs.build.outputs.TagFor7000072 }} - FW9Branch: ${{ needs.build.outputs.FW9Branch }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a0a4d450..6eb33fd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,9 +6,6 @@ on: MsBuildVersion: required: true type: string - FW9Branch: - required: true - type: string TagFor7000072: required: true type: string @@ -18,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3.0.2 with: fetch-depth: 0 @@ -31,12 +28,11 @@ jobs: - name: Tag release branches if: github.event_name == 'push' && github.ref == 'refs/heads/live' env: - FW9Branch: ${{ inputs.FW9Branch }} TAG72: ${{ inputs.TagFor7000072 }} run: | git config --global user.name "github-actions" git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" - git tag -f -a -m "Release ${TAG72}" "${TAG72}" "refs/remotes/origin/${FW9Branch}" + git tag -f -a -m "Release ${TAG72}" "${TAG72}" "refs/remotes/origin/live" git push -v origin "${TAG72}" - name: Download build artifacts @@ -54,16 +50,6 @@ jobs: - name: Verify that uncompress step worked run: ls -lR tarball - - name: Restore executable bits - # GitHub artifacts system strips executable bits, so restore them here - # TODO: Now that we upload .tar.gz format, no longer needed since tar format preserves Unix permissions. Verify, then remove this section - run: | - chmod +x tarball/lfmerge*/usr/bin/lfmerge - chmod +x tarball/lfmerge*/usr/bin/lfmergeqm - chmod +x tarball/lfmerge*/usr/lib/lfmerge/*/chorusmerge - chmod +x tarball/lfmerge*/usr/lib/lfmerge/*/startlfmerge - chmod +x tarball/lfmerge*/usr/lib/lfmerge/*/Mercurial/hg - - name: Login to GHCR if: github.event_name == 'push' && github.ref == 'refs/heads/live' uses: docker/login-action@v1 diff --git a/.vscode/launch.json b/.vscode/launch.json index b2826621..8408899d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -34,31 +34,10 @@ } }, */ - { - "name": "Launch unit tests", - "env": { - // Display on main display :0, or on a Xephyr :2. - "DISPLAY": ":0", - // When launching LfMerge, use this mono. - "PATH": "/opt/mono5-sil/bin:${env:PATH}", - // Path to the rest of the necessary enviromnent settings for LfMerge. This is handled by a mono wrapper script in /opt. - "MONO_ENVIRON": "${workspaceRoot}/environ" - }, - "type": "mono", - "request": "launch", - "program": "${workspaceRoot}/packages/NUnit.Runners.Net4.2.6.4/tools/nunit.exe", - "args": [ - "LfMerge.Core.Tests.dll" - ], - "cwd": "${workspaceRoot}/output/Debug", - "console": "integratedTerminal" - }, - { - "name": "Attach", - "type": "mono", - "request": "attach", - "address": "localhost", - "port": 55555 - } - ] + { + "name": ".NET Core Attach", + "type": "coreclr", + "request": "attach", + }, + ] } \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000..90df8e9d --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,47 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "build", + "command": "dotnet", + "type": "shell", + "args": [ + "build", + // Ask dotnet build to generate full paths for file names. + "/property:GenerateFullPaths=true", + // Do not generate summary otherwise it leads to duplicate errors in Problems panel + "/consoleloggerparameters:NoSummary" + ], + "group": "build", + "presentation": { + "reveal": "silent" + }, + "problemMatcher": "$msCompile" + }, + { + "label": ".NET Core Test with debugger", + "type": "process", + "isBackground": true, + "command": "dotnet", + "args": [ "test" ], + "options": + { + "cwd": "${workspaceFolder}", + "env": + { + "VSTEST_HOST_DEBUG": "1" + }, + }, + "group": "test", "presentation": + { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared" + }, + "problemMatcher": [] + }, + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7c082f62..4843960c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,51 +9,35 @@ RUN test -n "$BUILDER_UID" ENV BUILDER_UID="$BUILDER_UID" # # Build as a non-root user -RUN useradd -u "${BUILDER_UID:-DEFAULT_BUILDER_UID}" -d /home/builder -g users -G www-data,fieldworks,systemd-journal -m -s /bin/bash builder ; \ +RUN useradd -u "${BUILDER_UID:-DEFAULT_BUILDER_UID}" -d /home/builder -g users -G www-data,fieldworks -m -s /bin/bash builder ; \ echo "builder ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers; \ chown -R builder:users /build -# Any setup unique to the various builds goes in one of these four images -FROM lfmerge-builder-base AS lfmerge-build-7000068 -ENV DbVersion=7000068 -ENV DBVERSIONPATH=/usr/lib/lfmerge/7000068 -ENV RUN_UNIT_TESTS=0 -ENV UPDATE_ASSEMBLYINFO_BY_SCRIPT=1 -ENV NUNIT_VERSION_MAJOR=2 -# To run specific unit tests, set TEST_SPEC env var, e.g.: -# ENV TEST_SPEC=LfMerge.Core.Tests.Actions.SynchronizeActionTests.SynchronizeAction_CustomReferenceAtomicField_DoesNotThrowExceptionDuringSync - -FROM lfmerge-builder-base AS lfmerge-build-7000069 -ENV DbVersion=7000069 -ENV DBVERSIONPATH=/usr/lib/lfmerge/7000069 -ENV RUN_UNIT_TESTS=0 -ENV UPDATE_ASSEMBLYINFO_BY_SCRIPT=1 -ENV NUNIT_VERSION_MAJOR=2 -# ENV TEST_SPEC=LfMerge.Core.Tests.Actions.SynchronizeActionTests.SynchronizeAction_CustomReferenceAtomicField_DoesNotThrowExceptionDuringSync - -FROM lfmerge-builder-base AS lfmerge-build-7000070 -ENV DbVersion=7000070 -ENV DBVERSIONPATH=/usr/lib/lfmerge/7000070 -ENV RUN_UNIT_TESTS=0 -ENV UPDATE_ASSEMBLYINFO_BY_SCRIPT=1 -ENV NUNIT_VERSION_MAJOR=2 -# ENV TEST_SPEC=LfMerge.Core.Tests.Actions.SynchronizeActionTests.SynchronizeAction_CustomReferenceAtomicField_DoesNotThrowExceptionDuringSync +# Any setup unique to one specific DbVersion build goes in one of the images below FROM lfmerge-builder-base AS lfmerge-build-7000072 ENV DbVersion=7000072 ENV DBVERSIONPATH=/usr/lib/lfmerge/7000072 ENV RUN_UNIT_TESTS=0 -ENV UPDATE_ASSEMBLYINFO_BY_SCRIPT=0 ENV NUNIT_VERSION_MAJOR=3 # ENV TEST_SPEC=LfMerge.Core.Tests.Actions.SynchronizeActionTests.SynchronizeAction_CustomReferenceAtomicField_DoesNotThrowExceptionDuringSync FROM lfmerge-build-${DbVersion} AS lfmerge-build +# So unit tests can run +RUN mkdir -p /var/lib/languageforge/lexicon/sendreceive/ \ + && cd /var/lib/languageforge/lexicon/sendreceive/ && mkdir Templates state editqueue syncqueue webwork && cd - \ + && chown -R builder:users /var/lib/languageforge/lexicon/sendreceive + USER builder +WORKDIR /home/builder/repo +# Git repo should be mounted under /home/builder/repo when run +# E.g., `docker run --mount type=bind,source="$(pwd)",target=/home/builder/repo + +# NuGet package cache dir should be bind-mounted from home of running user +# E.g., `docker run --mount type=bind,source="${HOME}/.nuget/packages",target=/home/builder/.nuget/packages +RUN mkdir -p /home/builder/.nuget/packages -# Git repo should be mounted under ${HOME}/packages/lfmerge when run -# E.g., `docker run --mount type=bind,source="$(pwd)",target=/home/builder/packages/lfmerge` -RUN mkdir -p /home/builder/repo /home/builder/packages/lfmerge -CMD /home/builder/repo/docker/scripts/build-and-test.sh ${DbVersion} +CMD docker/scripts/build-and-test.sh ${DbVersion} # CMD doesn't actually run the script, it just gives `docker run` a default. # So it's okay for the Git repo to not be mounted yet. diff --git a/Dockerfile.builder-base b/Dockerfile.builder-base index ce4b5df2..7a32f2e4 100644 --- a/Dockerfile.builder-base +++ b/Dockerfile.builder-base @@ -5,16 +5,9 @@ WORKDIR /build/lfmerge ENV DEBIAN_FRONTEND=noninteractive ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 -RUN apt-get update && apt-get install -y gnupg -COPY sil-packages-key.gpg . -COPY sil-packages-testing-key.gpg . -RUN apt-key add sil-packages-key.gpg -RUN apt-key add sil-packages-testing-key.gpg -RUN echo 'deb http://linux.lsdev.sil.org/ubuntu bionic main' > /etc/apt/sources.list.d/llso-experimental.list -RUN echo 'deb http://linux.lsdev.sil.org/ubuntu bionic-experimental main' >> /etc/apt/sources.list.d/llso-experimental.list # Dependencies from Debian "control" file -RUN apt-get update && apt-get install -y sudo debhelper devscripts cli-common-dev iputils-ping cpp python-dev pkg-config mono5-sil mono5-sil-msbuild libicu-dev lfmerge-fdo && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y curl sudo iputils-ping cpp python-is-python2 python2-dev pkg-config libicu-dev && rm -rf /var/lib/apt/lists/* # Ensure fieldworks group exists in case lfmerge-fdo package didn't install it, and that www-data is part of that group # Also ensure that www-data has a .local dir in its home directory (/var/www) since some of lfmerge's dependencies assume that $HOME/.local exists diff --git a/Dockerfile.finalresult b/Dockerfile.finalresult index 8d595c8a..48a541b7 100644 --- a/Dockerfile.finalresult +++ b/Dockerfile.finalresult @@ -6,18 +6,12 @@ FROM ghcr.io/sillsdev/lfmerge-base:runtime # install LFMerge prerequisites # tini - PID 1 handler # python - required by Mercurial, which is bundled in the LFMerge installation -# rsyslog - lfmerge logs to rsyslog and expects this to exist -# rsyslog customizations (imklog reads kernel messages, which isn't allowed or desired in Docker containers) -# logrotate - ensure logs won't grow without bound, and that old logs get cleaned up automatically # iputils-ping - Chorus (part of LFMerge) requires the "ping" command to be available on the command line # less - so we can read syslog during manual debugging of issues # vim-tiny - so we can edit state files (to change HOLD to IDLE) during manual debugging of issues -RUN curl -L http://linux.lsdev.sil.org/downloads/sil-testing.gpg | apt-key add - \ -&& echo "deb http://linux.lsdev.sil.org/ubuntu bionic main" > /etc/apt/sources.list.d/linux-lsdev-sil-org.list \ -&& apt-get update \ -&& apt-get install --yes --no-install-recommends tini python rsyslog logrotate iputils-ping inotify-tools less vim-tiny \ -&& rm -rf /var/lib/apt/lists/* \ -&& sed -i '/load="imklog"/s/^/#/' /etc/rsyslog.conf +RUN apt-get update \ + && apt-get install --yes --no-install-recommends tini python iputils-ping inotify-tools less vim-tiny \ + && rm -rf /var/lib/apt/lists/* ADD tarball/lfmerge* / @@ -25,8 +19,6 @@ RUN mkdir -m 02775 -p /var/lib/languageforge/lexicon/sendreceive/syncqueue /var/ RUN install -d -o www-data -g www-data -m 02775 /var/run/lfmerge -# TODO: Turn this into environment variables, because we want to be able to just set env vars in k8s config and restart the container -COPY lfmerge.conf /etc/languageforge/conf/sendreceive.conf COPY sudoers.d.lfmerge.conf /etc/sudoers.d/lfmerge COPY mercurial-cacerts.rc /etc/mercurial/hgrc COPY lfmergeqm-background.sh / diff --git a/Dockerfile.runtime-base b/Dockerfile.runtime-base index 57d06acf..3d1406a7 100644 --- a/Dockerfile.runtime-base +++ b/Dockerfile.runtime-base @@ -5,16 +5,9 @@ WORKDIR /build/lfmerge ENV DEBIAN_FRONTEND=noninteractive ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 -RUN apt-get update && apt-get install -y gnupg -COPY sil-packages-key.gpg . -COPY sil-packages-testing-key.gpg . -RUN apt-key add sil-packages-key.gpg -RUN apt-key add sil-packages-testing-key.gpg -RUN echo 'deb http://linux.lsdev.sil.org/ubuntu bionic main' > /etc/apt/sources.list.d/llso-experimental.list -RUN echo 'deb http://linux.lsdev.sil.org/ubuntu bionic-experimental main' >> /etc/apt/sources.list.d/llso-experimental.list # Dependencies from Debian "control" file -RUN apt-get update && apt-get install -y sudo debhelper devscripts cli-common-dev iputils-ping cpp python-dev pkg-config mono5-sil mono5-sil-msbuild libicu-dev lfmerge-fdo && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y curl sudo iputils-ping cpp python-is-python2 python2-dev pkg-config libicu-dev && rm -rf /var/lib/apt/lists/* # Ensure fieldworks group exists in case lfmerge-fdo package didn't install it, and that www-data is part of that group # Also ensure that www-data has a .local dir in its home directory (/var/www) since some of lfmerge's dependencies assume that $HOME/.local exists diff --git a/LfMerge.sln b/LfMerge.sln index e7d87188..090e3358 100644 --- a/LfMerge.sln +++ b/LfMerge.sln @@ -6,8 +6,6 @@ MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{4D9087DE-A03C-4B46-AE3A-CEB3F5A5D146}" ProjectSection(SolutionItems) = preProject environ = environ - build\LfMerge.proj = build\LfMerge.proj - build\NuGet.targets = build\NuGet.targets README.md = README.md CHANGELOG.md = CHANGELOG.md EndProjectSection @@ -31,105 +29,54 @@ EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU - Debug7000068|Any CPU = Debug7000068|Any CPU - Debug7000069|Any CPU = Debug7000069|Any CPU - Debug7000070|Any CPU = Debug7000070|Any CPU Debug7000072|Any CPU = Debug7000072|Any CPU Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug7000068|Any CPU.ActiveCfg = Debug|Any CPU - {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug7000068|Any CPU.Build.0 = Debug|Any CPU - {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug7000069|Any CPU.ActiveCfg = Debug|Any CPU - {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug7000069|Any CPU.Build.0 = Debug|Any CPU - {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug7000070|Any CPU.ActiveCfg = Debug|Any CPU - {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug7000070|Any CPU.Build.0 = Debug|Any CPU {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E79BC8D-9AE4-4B2C-ACFB-D8C1B2BC3D4F}.Release|Any CPU.Build.0 = Release|Any CPU {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug7000068|Any CPU.ActiveCfg = Debug|Any CPU - {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug7000068|Any CPU.Build.0 = Debug|Any CPU - {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug7000069|Any CPU.ActiveCfg = Debug|Any CPU - {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug7000069|Any CPU.Build.0 = Debug|Any CPU - {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug7000070|Any CPU.ActiveCfg = Debug|Any CPU - {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug7000070|Any CPU.Build.0 = Debug|Any CPU {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Release|Any CPU.ActiveCfg = Release|Any CPU {B514E195-C83F-4CC9-A9BF-7F5A31952AAF}.Release|Any CPU.Build.0 = Release|Any CPU {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug7000068|Any CPU.ActiveCfg = Debug|Any CPU - {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug7000068|Any CPU.Build.0 = Debug|Any CPU - {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug7000069|Any CPU.ActiveCfg = Debug|Any CPU - {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug7000069|Any CPU.Build.0 = Debug|Any CPU - {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug7000070|Any CPU.ActiveCfg = Debug|Any CPU - {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug7000070|Any CPU.Build.0 = Debug|Any CPU {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Release|Any CPU.ActiveCfg = Release|Any CPU {FF82CEC0-353A-4E79-AB7E-6AFEF1F15EC2}.Release|Any CPU.Build.0 = Release|Any CPU {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug7000068|Any CPU.ActiveCfg = Debug|Any CPU - {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug7000068|Any CPU.Build.0 = Debug|Any CPU - {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug7000069|Any CPU.ActiveCfg = Debug|Any CPU - {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug7000069|Any CPU.Build.0 = Debug|Any CPU - {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug7000070|Any CPU.ActiveCfg = Debug|Any CPU - {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug7000070|Any CPU.Build.0 = Debug|Any CPU {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Release|Any CPU.ActiveCfg = Release|Any CPU {AFD8D8A6-2D98-482F-B367-6C0268B61154}.Release|Any CPU.Build.0 = Release|Any CPU {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug|Any CPU.Build.0 = Debug|Any CPU - {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug7000068|Any CPU.ActiveCfg = Debug|Any CPU - {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug7000068|Any CPU.Build.0 = Debug|Any CPU - {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug7000069|Any CPU.ActiveCfg = Debug|Any CPU - {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug7000069|Any CPU.Build.0 = Debug|Any CPU - {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug7000070|Any CPU.ActiveCfg = Debug|Any CPU - {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug7000070|Any CPU.Build.0 = Debug|Any CPU {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Release|Any CPU.ActiveCfg = Release|Any CPU {40EC1EB4-F1EA-4FD7-9A4E-1ACA5BA31FC3}.Release|Any CPU.Build.0 = Release|Any CPU {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug7000068|Any CPU.ActiveCfg = Debug|Any CPU - {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug7000068|Any CPU.Build.0 = Debug|Any CPU - {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug7000069|Any CPU.ActiveCfg = Debug|Any CPU - {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug7000069|Any CPU.Build.0 = Debug|Any CPU - {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug7000070|Any CPU.ActiveCfg = Debug|Any CPU - {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug7000070|Any CPU.Build.0 = Debug|Any CPU {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Release|Any CPU.ActiveCfg = Release|Any CPU {DED60A67-66FC-44DD-8AD5-67E470D74A53}.Release|Any CPU.Build.0 = Release|Any CPU {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug7000068|Any CPU.ActiveCfg = Debug|Any CPU - {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug7000068|Any CPU.Build.0 = Debug|Any CPU - {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug7000069|Any CPU.ActiveCfg = Debug|Any CPU - {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug7000069|Any CPU.Build.0 = Debug|Any CPU - {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug7000070|Any CPU.ActiveCfg = Debug|Any CPU - {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug7000070|Any CPU.Build.0 = Debug|Any CPU {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Release|Any CPU.ActiveCfg = Release|Any CPU {3152A333-A741-4EE7-A791-DFE0E4C5E2AA}.Release|Any CPU.Build.0 = Release|Any CPU {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000068|Any CPU.ActiveCfg = Debug|Any CPU - {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000068|Any CPU.Build.0 = Debug|Any CPU - {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000069|Any CPU.ActiveCfg = Debug|Any CPU - {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000069|Any CPU.Build.0 = Debug|Any CPU - {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000070|Any CPU.ActiveCfg = Debug|Any CPU - {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000070|Any CPU.Build.0 = Debug|Any CPU {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000072|Any CPU.ActiveCfg = Debug|Any CPU {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Debug7000072|Any CPU.Build.0 = Debug|Any CPU {28882F30-358B-4E1C-A934-076D9EE6ACFC}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -141,194 +88,4 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {34938BD0-DBBC-4C1A-B02A-2571C0783AD5} EndGlobalSection - GlobalSection(MonoDevelopProperties) = preSolution - Policies = $0 - $0.StandardHeader = $1 - $1.Text = @Copyright (c) ${Year} SIL International\nThis software is licensed under the MIT license (http://opensource.org/licenses/MIT) - $1.IncludeInNewFiles = True - $0.TextStylePolicy = $8 - $2.inheritsSet = null - $2.scope = text/x-csharp - $0.CSharpFormattingPolicy = $3 - $3.PropertyBraceStyle = NextLine - $3.PropertyGetBraceStyle = NextLine - $3.PropertySetBraceStyle = NextLine - $3.EventBraceStyle = NextLine - $3.EventAddBraceStyle = NextLine - $3.EventRemoveBraceStyle = NextLine - $3.StatementBraceStyle = NextLine - $3.ElseNewLinePlacement = NewLine - $3.CatchNewLinePlacement = NewLine - $3.FinallyNewLinePlacement = NewLine - $3.BeforeMethodDeclarationParentheses = False - $3.BeforeMethodCallParentheses = False - $3.BeforeConstructorDeclarationParentheses = False - $3.BeforeIndexerDeclarationBracket = False - $3.BeforeDelegateDeclarationParentheses = False - $3.AfterDelegateDeclarationParameterComma = True - $3.NewParentheses = False - $3.SpacesBeforeBrackets = False - $3.AlignToFirstMethodDeclarationParameter = False - $3.AlignToFirstIndexerDeclarationParameter = False - $3.inheritsSet = Mono - $3.inheritsScope = text/x-csharp - $3.scope = text/x-csharp - $0.DotNetNamingPolicy = $4 - $4.DirectoryNamespaceAssociation = PrefixedHierarchical - $4.ResourceNamePolicy = FileName - $5.FileWidth = 98 - $5.TabsToSpaces = False - $5.inheritsSet = VisualStudio - $5.inheritsScope = text/plain - $5.scope = text/plain - $6.inheritsSet = null - $6.scope = application/config+xml - $0.XmlFormattingPolicy = $9 - $7.inheritsSet = null - $7.scope = application/config+xml - $8.inheritsSet = null - $8.scope = application/xml - $9.inheritsSet = Mono - $9.inheritsScope = application/xml - $9.scope = application/xml - $0.NameConventionPolicy = $10 - $10.Rules = $11 - $11.NamingRule = $38 - $12.Name = Namespaces - $12.AffectedEntity = Namespace - $12.VisibilityMask = VisibilityMask - $12.NamingStyle = PascalCase - $12.IncludeInstanceMembers = True - $12.IncludeStaticEntities = True - $13.Name = Types - $13.AffectedEntity = Class, Struct, Enum, Delegate - $13.VisibilityMask = VisibilityMask - $13.NamingStyle = PascalCase - $13.IncludeInstanceMembers = True - $13.IncludeStaticEntities = True - $14.Name = Interfaces - $14.RequiredPrefixes = $15 - $15.String = I - $14.AffectedEntity = Interface - $14.VisibilityMask = VisibilityMask - $14.NamingStyle = PascalCase - $14.IncludeInstanceMembers = True - $14.IncludeStaticEntities = True - $16.Name = Attributes - $16.RequiredSuffixes = $17 - $17.String = Attribute - $16.AffectedEntity = CustomAttributes - $16.VisibilityMask = VisibilityMask - $16.NamingStyle = PascalCase - $16.IncludeInstanceMembers = True - $16.IncludeStaticEntities = True - $18.Name = Event Arguments - $18.RequiredSuffixes = $19 - $19.String = EventArgs - $18.AffectedEntity = CustomEventArgs - $18.VisibilityMask = VisibilityMask - $18.NamingStyle = PascalCase - $18.IncludeInstanceMembers = True - $18.IncludeStaticEntities = True - $20.Name = Exceptions - $20.RequiredSuffixes = $21 - $21.String = Exception - $20.AffectedEntity = CustomExceptions - $20.VisibilityMask = VisibilityMask - $20.NamingStyle = PascalCase - $20.IncludeInstanceMembers = True - $20.IncludeStaticEntities = True - $22.Name = Methods - $22.AffectedEntity = Methods - $22.VisibilityMask = VisibilityMask - $22.NamingStyle = PascalCase - $22.IncludeInstanceMembers = True - $22.IncludeStaticEntities = True - $23.Name = Static Readonly Fields - $23.AffectedEntity = ReadonlyField - $23.VisibilityMask = Internal, Protected, Public - $23.NamingStyle = PascalCase - $23.IncludeInstanceMembers = False - $23.IncludeStaticEntities = True - $24.Name = Fields (Non Private) - $24.AffectedEntity = Field - $24.VisibilityMask = Internal, Protected, Public - $24.NamingStyle = PascalCase - $24.IncludeInstanceMembers = True - $24.IncludeStaticEntities = True - $25.Name = ReadOnly Fields (Non Private) - $25.AffectedEntity = ReadonlyField - $25.VisibilityMask = Internal, Protected, Public - $25.NamingStyle = PascalCase - $25.IncludeInstanceMembers = True - $25.IncludeStaticEntities = False - $26.Name = Fields (Private) - $26.RequiredPrefixes = $27 - $27.String = _ - $26.AffectedEntity = Field, ReadonlyField - $26.VisibilityMask = Private - $26.NamingStyle = CamelCase - $26.IncludeInstanceMembers = True - $26.IncludeStaticEntities = False - $28.Name = Static Fields (Private) - $28.AffectedEntity = Field - $28.VisibilityMask = Private - $28.NamingStyle = CamelCase - $28.IncludeInstanceMembers = False - $28.IncludeStaticEntities = True - $29.Name = ReadOnly Fields (Private) - $29.RequiredPrefixes = $30 - $30.String = _ - $29.AffectedEntity = ReadonlyField - $29.VisibilityMask = Private - $29.NamingStyle = CamelCase - $29.IncludeInstanceMembers = True - $29.IncludeStaticEntities = False - $31.Name = Constant Fields - $31.AffectedEntity = ConstantField - $31.VisibilityMask = VisibilityMask - $31.NamingStyle = PascalCase - $31.IncludeInstanceMembers = True - $31.IncludeStaticEntities = True - $32.Name = Properties - $32.AffectedEntity = Property - $32.VisibilityMask = VisibilityMask - $32.NamingStyle = PascalCase - $32.IncludeInstanceMembers = True - $32.IncludeStaticEntities = True - $33.Name = Events - $33.AffectedEntity = Event - $33.VisibilityMask = VisibilityMask - $33.NamingStyle = PascalCase - $33.IncludeInstanceMembers = True - $33.IncludeStaticEntities = True - $34.Name = Enum Members - $34.AffectedEntity = EnumMember - $34.VisibilityMask = VisibilityMask - $34.NamingStyle = PascalCase - $34.IncludeInstanceMembers = True - $34.IncludeStaticEntities = True - $35.Name = Parameters - $35.AffectedEntity = Parameter - $35.VisibilityMask = VisibilityMask - $35.NamingStyle = CamelCase - $35.IncludeInstanceMembers = True - $35.IncludeStaticEntities = True - $36.Name = Type Parameters - $36.RequiredPrefixes = $37 - $37.String = T - $36.AffectedEntity = TypeParameter - $36.VisibilityMask = VisibilityMask - $36.NamingStyle = PascalCase - $36.IncludeInstanceMembers = True - $36.IncludeStaticEntities = True - $38.Name = Fields (Private) - $38.RequiredPrefixes = $39 - $39.String = _ - $38.AffectedEntity = None - $38.VisibilityMask = Private - $38.NamingStyle = PascalCase - $38.IncludeInstanceMembers = True - $38.IncludeStaticEntities = False - EndGlobalSection EndGlobal diff --git a/README.md b/README.md index a504fd93..ed3764b2 100644 --- a/README.md +++ b/README.md @@ -14,21 +14,34 @@ You'll need Docker installed, as well as GNU Parallel to use the parallel-build ## Development -First, a word about the branching scheme of this repository and how it relates to FieldWorks. LfMerge supports FieldWorks versions 8.2 and later. FieldWorks has had different data models, represented by a six-digit integer called DbVersion. The data models for FieldWorks 8.x had DbVersions 7000068 through 7000070. DbVersion 7000071 was never released in anything but alpha builds of FieldWorks, and LfMerge does not support it. FieldWorks 9.0 and above uses DbVersion 7000072. +The `master` branch in this repository is where most development happens (it would be called `develop` if we were strictly following a Git-Flow naming scheme). The `live` branch is what release images are built from. Pushing to the `live` branch will build the Docker image, then if the build succeeds, it will tag the built commit with the version number with a `v` prefix (e.g., `v2.0.123`) and tag the Docker image with the unprefixed version number (e.g., `lfmerge:2.0.123`). -There are two main branches in the LfMerge repo, `master` and `fieldworks8-master`. That's because key parts of the FieldWorks API changed between FW 8 and FW 9, such as FDO (FieldWorks Data Objects) being renamed to LCM (Language and Culture Model) and large parts of the FDO API being moved to a library called liblcm. The process of acquiring FieldWorks DLLs also changed between FW 8 and FW 9. FW 8 DLLs (and other libraries that work with FW 8, such as Chorus) were never packaged as NuGet packages, so we have to download those DLLs as build artifacts from various TeamCity builds. FW 9 and its supporting libraries, however, are available as NuGet packages. So the build process for `master` and `fieldworks8-master` is slightly different. More on that in the Building section below. +The `live` branch is expected to be **fast-forwarded**, with no merge commits, from the `master` branch. When you want to push a new release, please do: -In addition to `master` and `fieldworks8-master`, there are also `qa` and `fieldworks8-qa` branches, and `live` and `fieldworks8-live` branches. The two `live` branches are the ones that the release is built from, while the `qa` branch is for prerelease builds. One consequence of this is that **most pull requests need to be doubled up**. When you create a feature branch called `feature/foo` and make a PR from it (against `master`), you'll need to make a similar feature branch and call it `feature/foo-fw8`, then create another PR from that branch against `fieldworks8-master`. Once someone has reviewed the PR against `master`, you can merge **both** PRs unless the reviewer specifically says not to. +1. `git checkout master` +1. `git pull --ff-only` to make sure you have the latest merged PRs +1. Deal with any merge conflicts, then run `git pull --ff-only` again if there were any conflicts +1. Once there are no merge conflicts, `git checkout live` +1. `git merge --ff-only master` +1. If no merge conflicts, `git push live` ## Building -For each DbVersion that LfMerge supports, we build a different lfmerge binary. DbVersions 7000068 through 7000070 are built from FW 8 branches (`fieldworks8-master` or `feature/foo-fw8`), while DbVersion 7000072 is built from an FW 9 branch (`master` or `feature/foo`). There is a script called `pbuild.sh` (for "parallel build") that will handle all the complexity of the build process for you. It will run the build for each DbVersion in a Docker container, using a common Docker build image, and then copy the final results into a directory called `tarball`. Finally, it will run a Docker build that will take the files in the `tarball` directory and turn then into a Docker image for `lfmerge`. By default, this Docker image will be tagged `ghcr.io/sillsdev/lfmerge:latest`, the same tag as the tag built by the GitHub Actions workflow. - -**Normally you will run `pbuild.sh` with no parameters.** It will look at the Git branch you have checked out, determine whether that branch is a branch based on FieldWorks 9 (DbVersion 7000072 or later) or FieldWorks 8 (DbVersions 7000068 through 7000070), and calculate the corresponding branch in the other FW version. E.g., if you're on `master` the corresponding branch will be `fieldworks8-master`. If you want different behavior, for example you want to run a build from `feature/foo` but use `fieldworks8-master` as the FW8 branch instead of `feature/foo-fw8`, then you can pass a parameter to `pbbuild.sh` to set what the corresponding branch should be. E.g. with `feature/foo` checked out, run `pbuild.sh fieldworks8-master` and you'll get a build where the DbVersion 7000072 binaries were built from the `feature/foo` branch, but where the DbVersion 7000068-7000070 binaries were built from `fieldworks8-master`. **You should rarely need this**, but it's sometimes helpful when you're trying to track down a bug that appears only in the FW 8 builds but not the FW 9 build, or vice-versa. +For each DbVersion that LfMerge supports, we build a different lfmerge binary. We used to support DbVersions 7000068 through 7000070, which correspond to various versions of FieldWorks 8.x, but we now only support FieldWorks 9.x. The only DbVersion found in FieldWorks 9.x is currently (as of August 2022) 7000072. There is a script called `pbuild.sh` (for "parallel build") that will handle building all currently-supported DbVersions. It will run the build for each DbVersion in a Docker container, using a common Docker build image, and then copy the final results into a directory called `tarball`. Finally, it will run a Docker build that will take the files in the `tarball` directory and turn then into a Docker image for `lfmerge`. By default, this Docker image will be tagged `ghcr.io/sillsdev/lfmerge:latest`, while the GitHub Actions workflow will produce a specific version number tag as well as `latest`, i.e. the GHA workflow will tag `lfmerge:2.0.123` as well as `lfmerge:latest`. ## Testing locally -The image that `pbuild.sh` produces is tagged with the same image tag as the one built by the official GitHub Actions workflow. This means that if you're running Language Forge locally via the Makefile in Language Forge's `docker` directory, you should be able to simply run `make` and the `lfmerge` container will be re-created with your local build. You can then do a Send/Receive via your `localhost` copy of Language Forge and check the results. +The image that `pbuild.sh` produces is tagged with the same image tag as the one built by the official GitHub Actions workflow, but with a `latest` tag instead of a versino number. This means that if you're running Language Forge locally via the Makefile in Language Forge's `docker` directory, you should be able to simply edit the `docker/lfmerge/Dockerfile` in Language Forge. Make sure that file specifies `lfmerge:latest` rather than `lfmerge:20..123`. Then run `make` and the `lfmerge` container will be re-created with your local build. You can then do a Send/Receive via your `localhost` copy of Language Forge and check the results. + +## Debugging unit tests in VS Code + +Debugging unit tests in VS Code is a two-step process. First, choose the "Run Test Task" command in VS Code. This task is configured to start the `dotnet test` process in debug mode, which means that it will set itself up (building the project if necessary), then pause itself waiting for a debugger to attach. Once it has paused itself, go to the debug tab (Crtl+Shift+D) and choose the "Attach" configuration. When prompted, enter the PID printed on the test console. (There will be two, one for LfMerge.Tests and one for LfMerge.Core.Tests; most of the tests are in LfMerge.Core so that's the one you'll need most of the time). The debugger will attach to the test process, which is still in a paused state. Set any breakpoints you want to hit in the unit tests, then unpause the debugger and it will start running the unit tests. + +To run specific unit tests instead of the whole suite, edit the `.vscode/tasks.json` file and change the launch args from `[ "test" ]` to something containing a filter, e.g. + +`[ "test", "--filter", "FullyQualifiedName~GetCustomFieldForThisCmObject" ],` + +The `~` in `FullyQualifiedName~` means "contains". For more possible filters you could use, see https://aka.ms/vstest-filtering ## Logs diff --git a/build/LfMerge.proj b/build/LfMerge.proj deleted file mode 100644 index 2a10c516..00000000 --- a/build/LfMerge.proj +++ /dev/null @@ -1,111 +0,0 @@ - - - $(MSBuildProjectDirectory)/.. - LfMerge.sln - $(RootDir)/$(Solution) - $(RootDir)/packages - LfMerge - lfmerge - Release - true - false - 2.6.4 - true - false - $(RootDir)/output/$(Configuration)/TestResults.xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/build/NuGet.targets b/build/NuGet.targets deleted file mode 100644 index ddec48dd..00000000 --- a/build/NuGet.targets +++ /dev/null @@ -1,11 +0,0 @@ - - - - $(MSBuildProjectDirectory)/.. - $(RootDir)/packages - - - - - diff --git a/build/install-deps b/build/install-deps deleted file mode 100755 index 4d6c49b2..00000000 --- a/build/install-deps +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -e -# Copyright (c) 2016 SIL International -# This software is licensed under the MIT License (http://opensource.org/licenses/MIT) -# -# Install required dependencies (as defined in the package control file) - -sudo apt-get update -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y equivs devscripts -sudo DEBIAN_FRONTEND=noninteractive mk-build-deps --install --remove \ - --tool "apt-get --no-install-recommends -y" $(dirname "$0")/../debian/control -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - $(apt-cache depends lfmerge-build-deps | grep Depends | cut -f 4 -d" ") diff --git a/build/package.json b/build/package.json deleted file mode 100644 index 5337536f..00000000 --- a/build/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "deploy-live.gulpfile.js", - "description": "Gulpfile to download latest live .deb and deploy it on live server", - "devDependencies": { - "gulp-download": "0.0.1", - "gulp-fn": "0.0.0", - "gulp-json-transform": "^0.3.2", - "gulp-rename": "^1.2.2", - "gulp-savefile": "^0.1.1", - "gulp-shell": "^0.5.2" - } -} diff --git a/calculate-branches.sh b/calculate-branches.sh index 98c47d2f..2b1de6a9 100755 --- a/calculate-branches.sh +++ b/calculate-branches.sh @@ -6,80 +6,6 @@ HEAD_BRANCH=${HEAD_BRANCH#refs/heads/} GITHUB_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF:-$HEAD_BRANCH}} GITHUB_BRANCH=${GITHUB_BRANCH#refs/heads/} -if [ -n "${GITHUB_HEAD_REF}" ]; then - PR_BRANCH=${GITHUB_BRANCH} -fi - CURRENT_BRANCH=${GITHUB_BRANCH:-$HEAD_BRANCH} -PARENT_MAJOR_VERSION=$(git describe --long --match "v*" | cut -c1-2) - -# FW8 branches will have ancestors tagged v1.x, while FW9 branches will have ancestors tagged v2.x -if [ "x$PARENT_MAJOR_VERSION" = "xv2" ]; then - IS_FW9=true -else - IS_FW9="" -fi - -if [ "${IS_FW9}" ]; then - FW9_BUILD_BRANCH="${CURRENT_BRANCH}" - if [ "$1" ]; then - if [ "$1" = "--no-fw8" ]; then - BUILD_FW8=0 - FW8_BUILD_BRANCH="" - else - FW8_BUILD_BRANCH="$1" - fi - elif [ -n "${PR_BRANCH}" ]; then - # Find corresponding branch, if it exists, by adding "-fw8" - if git rev-parse -q --verify "${PR_BRANCH}-fw8" >/dev/null; then - FW8_BUILD_BRANCH="${PR_BRANCH}-fw8" - else - # Fall back to fieldworks8-master if no corresponding PR branch found - FW8_BUILD_BRANCH="fieldworks8-master" - fi - elif [ "${CURRENT_BRANCH}" = "master" -o "${CURRENT_BRANCH}" = "qa" -o "${CURRENT_BRANCH}" = "live" ]; then - FW8_BUILD_BRANCH="fieldworks8-${CURRENT_BRANCH}" - else - # Find corresponding branch, if it exists, by adding "-fw8" - if git rev-parse -q --verify "${CURRENT_BRANCH}-fw8" >/dev/null; then - FW8_BUILD_BRANCH="${CURRENT_BRANCH}-fw8" - else - # Fall back to fieldworks8-master if no corresponding PR branch found - FW8_BUILD_BRANCH="fieldworks8-master" - fi - fi -else - FW8_BUILD_BRANCH="${CURRENT_BRANCH}" - if [ "$1" ]; then - FW9_BUILD_BRANCH="$1" - elif [ -n "${PR_BRANCH}" ]; then - # Find corresponding branch, if it exists, by trimming "-fw8" - CANDIDATE=${PR_BRANCH%-fw8} - # If candidate is same as PR branch, this PR branch didn't match the naming scheme - if [ "x$CANDIDATE" = "x$PR_BRANCH" ]; then - FW9_BUILD_BRANCH="master" - elif git rev-parse -q --verify "${PR_BRANCH%-fw8}" >/dev/null; then - FW9_BUILD_BRANCH="${PR_BRANCH%-fw8}" - else - # Fall back to master if no corresponding PR branch found - FW9_BUILD_BRANCH="master" - fi - elif [ "${CURRENT_BRANCH}" = "fieldworks8-master" -o "${CURRENT_BRANCH}" = "fieldworks8-qa" -o "${CURRENT_BRANCH}" = "fieldworks8-live" ]; then - FW9_BUILD_BRANCH="${CURRENT_BRANCH##fieldworks8-}" - else - # Find corresponding branch, if it exists, by trimming "-fw8" - CANDIDATE=${CURRENT_BRANCH%-fw8} - # If candidate is same as current, we're on a branch that didn't match the naming scheme - if [ "x$CANDIDATE" = "x$CURRENT_BRANCH" ]; then - FW9_BUILD_BRANCH="master" - elif git rev-parse -q --verify "${CURRENT_BRANCH%-fw8}" >/dev/null; then - FW9_BUILD_BRANCH="${CURRENT_BRANCH%-fw8}" - else - # Fall back to master if no corresponding PR branch found - FW9_BUILD_BRANCH="master" - fi - fi -fi - -echo FW8Branch="${FW8_BUILD_BRANCH}" -echo FW9Branch="${FW9_BUILD_BRANCH}" +echo FW9Branch="${CURRENT_BRANCH}" +# We no longer have an FW8Branch output diff --git a/docker/common.sh b/docker/common.sh deleted file mode 100755 index d350f582..00000000 --- a/docker/common.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash - -stderr() -{ - echo -e "${RED}$PROGRAM_NAME: $1${NC}" >&2 -} - -log() -{ - echo -e "${GREEN}$PROGRAM_NAME: $1${NC}" >&2 -} - -TRACE() -{ - echo "$@" - "$@" -} - -general_init() -{ - # currently supported and future Ubuntu versions - UBUNTU_DISTROS="xenial bionic focal groovy hirsute" - # currently supported LTS versions - UBUNTU_LTS_DISTROS="xenial bionic focal" - # no longer supported Ubuntu versions that live in old-releases.ubuntu.com - UBUNTU_OLDDISTROS="" - # We're no longer building packages for: precise quantal raring saucy trusty utopic vivid wily yakkety zesty artful cosmic disco eoan - - # Debian versions - DEBIAN_DISTROS="stretch buster bullseye" - - DISTRIBUTIONS_TO_PACKAGE="${dists_arg:-bionic}" - DISTS_TO_PROCESS="${supported_distros_arg:-xenial bionic focal}" - DISTRIBUTIONS=${DISTRIBUTIONS:-$UBUNTU_DISTROS $UBUNTU_OLDDISTROS $DEBIAN_DISTROS} - ARCHES_TO_PACKAGE="${arches_arg:-i386 amd64}" - ARCHES_TO_PROCESS="amd64 i386" - PACKAGING_ROOT="${PACKAGING_ROOT:-$HOME/packages}" - SCHROOTDIR=/var/lib/schroot/chroots - - UBUNTU_MIRROR=${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/} - UBUNTU_OLDMIRROR=${UBUNTU_OLDMIRROR:-http://old-releases.ubuntu.com/ubuntu/} - - if [ "$suite_name" = "main" ]; then - SUITE_NAME="" - else - SUITE_NAME="-${suite_name:-experimental}" - fi - - RESULTBASE=${WORKSPACE:-$PACKAGING_ROOT} - - # The version of mk-sbuild we use. We need a current version of that script because trying - # to build newer dists might have different requirements than the system provided version - # provides. - # 0.177 is the version included in Groovy - MKSBUILD_VERSION=0.177 - - RED='\033[0;31m' - GREEN='\033[0;32m' - NC='\033[0m' # No Color -} - -init() -{ - # Process arguments. - while (( $# )); do - case $1 in - # Process individual arguments here. Use shift and $1 to get an argument value. - # Example: -d) DEBUG=true ;; - # Example: --outfile) shift; OUTFILE=$1 ;; - # Example: *) echo "Unexpected argument: $1"; exit 1 ;; - --debkeyid) shift; debkeyid=$1 ;; - # Space-delimited list of releases. eg "precise raring" - --dists) shift; dists_arg=$1 ;; - # Space-delimited list of architectures. eg "amd64 i386" - --arches) shift; arches_arg=$1 ;; - # Comma-delimited list of non-default repositorydir=committish mapping (hash, tag, branch). eg "fwrepo/fw=2568e4f,fwrepo/fw/Localizations=linux/FieldWorks8.0.3-beta4,fwrepo/fw/DistFiles/Helps=origin/master" - --repository-committishes) shift; repository_committishes_arg=$1 ;; - # Don't upload packages at the end - --simulate-dput) dput_simulate="-s" ;; - --package-version-extension) shift; package_version_extension=$1 ;; - # Suite location: eg. experimental, updates, proposed, dictionary - --suite-name) shift; suite_name=$1 ;; - --main-package-name) shift; main_package_name_arg=$1 ;; - # Skip cleaning and updating local repository - --preserve-repository) preserve_repository_arg=true ;; - # Space-delimited list of binary packages to remove from debian/control file before creating source package - --omit-binary-packages) shift; omit_binary_packages_arg=$1 ;; - # For making release packages. Do not add a new entry to the changelog. Package versions will be as specified in the last changelog entry, without a nightly timestamp appended. - --preserve-changelog) preserve_changelog_arg=true ;; - # Omit uploading to llso. This parameter should be set when doing a release build on - # Jenkins because the package has to be downloaded, signed and manually uploaded. - --no-upload) no_upload=true ;; - # The distros we might possibly want to build - --supported-distros) shift; supported_distros_arg=$1 ;; - # The package version to use instead a version number based on the last version - # from the changelog. Any 0 will be replaced by the corresponding number from the - # changelog, e.g. passing 0.0.123.456 with a version from changelog of 3.1.2.3 - # will result in 3.1.123.456. - --package-version) shift; package_version=$1 ;; - # The subdirectory of the main repo, e.g. fw. Default is the current directory. - --main-repo-dir) shift; main_repo_dir=$1 ;; - # The name of the directory where the source code resides. Default is the name of the source package. - # This directory is relative to $repo_base_dir. - --source-code-subdir) shift; source_package_dir=$1 ;; - --no-package) no_package=true ;; - # append the argument to the package name. Only relevant for make-source. - --append-to-package) shift; append_to_package=$1;; - # use current directory to build source package instead of $repo_base_dir/${$source_package_name} - --build-in-place) build_in_place=true ;; - # use argument as delimiter that seperates the version number from the 'nightly' string - --nightly-delimiter) shift; nightlydelimeter=$1;; - --update) update=true;; - --help) helpScript; exit 0 ;; - *) stderr "Error: Unexpected argument \"$1\". Exiting." ; exit 1 ;; - esac - shift || (stderr "Error: The last argument is missing a value. Exiting."; false) || exit 2 - done - - general_init - - if [ -z "$no_package" ]; then - # set Debian/changelog environment - export DEBFULLNAME="${main_package_name_arg:-Unknown} Package Signing Key" - export DEBEMAIL='jenkins@sil.org' - - repo_base_dir=${WORKSPACE:-$PACKAGING_ROOT/$main_package_name_arg} - debian_path="debian" - source_package_name=$(dpkg-parsechangelog |grep ^Source:|cut -d' ' -f2) - fi - - if [ -n "$build_in_place" ]; then - if [ ! -d debian ]; then - if [ -n "$main_package_name_arg" ]; then - debian_path=${main_package_name_arg}/debian - else - packagedsc=$(ls *.dsc) - if [ $(wc -l $packagedsc) -gt 1 ]; then - stderr "Error: More than one .dsc file in directory and --main-package-name not specified" - exit 3 - fi - packagedsc=${packagedsc%%_*} - debian_path=${packagedsc}/debian - fi - fi - repo_base_dir=$(readlink -f $PWD) - source_package_name=$(cd $debian_path/.. && dpkg-parsechangelog |grep ^Source:|cut -d' ' -f2) - fi - - if [ -d ".hg" ]; then - VCS=hg - else - VCS=git - fi -} diff --git a/docker/compile-lfmerge.sh b/docker/compile-lfmerge.sh deleted file mode 100755 index 792e3ab7..00000000 --- a/docker/compile-lfmerge.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e -echo "Compiling LfMerge and running unit tests" -BUILD=Release . environ - -echo "Using $(which dotnet)" -dotnet build /t:CompileOnly /v:detailed /property:Configuration=Release build/LfMerge.proj -# TODO: Determine if this symlink is necessary -ln -sf ../Mercurial output/ -# TODO: Test all DLLs -# dotnet test -f net462 LfMerge.Core.Tests.dll diff --git a/docker/fw8-flexbridge.tar.xz b/docker/fw8-flexbridge.tar.xz deleted file mode 100644 index d83613a8..00000000 Binary files a/docker/fw8-flexbridge.tar.xz and /dev/null differ diff --git a/docker/make-source b/docker/make-source deleted file mode 100755 index ea41ae34..00000000 --- a/docker/make-source +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -e -# Create source package -# set supported distros and architectures - -PROGRAM_NAME="$(basename "$0")" -PROGRAM_DIR="$(dirname "$0")" - -function helpScript() { - cat << EOF - -$PROGRAM_NAME [--debkeyid ] [--build-in-place] - -Create a source package. - -Arguments: - --debkeyid - the GPG key to use to sign the package (instead of - trying to use Jenkins' GPG key) - --build-in-place - use the current directory as build directory instead - of \$repo_base_dir/\$source_package_name - -There are more optional parameters that can be set. See common.sh. - -NOTE: This help is incomplete. Please add the description of more options -that are useful and apply to this script. -EOF - exit 0 -} - -. ${PROGRAM_DIR}/common.sh -init "$@" - -log "Preparing" -mkdir -p "$repo_base_dir" -mkdir -p "$RESULTBASE" - -# change package name -timestamp=$(date -u +"%Y%m%dT%H%M%SZ") -latest_version_in_debian_changelog=$(dpkg-parsechangelog --show-field=Version) - -if [ "$(dpkg-parsechangelog --show-field=Distribution)" = "UNRELEASED" ]; then - unset DEBFULLNAME - export DEBEMAIL="$(dpkg-parsechangelog --show-field=Maintainer)" -fi - -if [ -n "$append_to_package" ]; then - NEWPACKAGENAME="--package=${source_package_name}${append_to_package}" - source_package_name=${source_package_name}${append_to_package} - ${PROGRAM_DIR}/change-package-name.pl $append_to_package - sed -e "s/^PACKAGENAME *=.*/&${append_to_package}/" -e "s/^APPEND *=.*/&${append_to_package}/" -i debian/rules - dch $NEWPACKAGENAME --newversion "${latest_version_in_debian_changelog}" --force-bad-version --nomultimaint "" -fi - -# Clean out old packages -rm -f "$repo_base_dir"/${source_package_name}_*.{dsc,build,changes,tar.*,log} - -if [ "$VCS" = "hg" ]; then - hash_of_current_commit=$(cd ${main_repo_dir:-.} && hg id -i 2>/dev/null |cut -c -12) -else - hash_of_current_commit=$(cd ${main_repo_dir:-.} && git rev-parse --short HEAD) -fi - -if [ -z "$package_version" ]; then - base_version=${code_version:-$latest_version_in_debian_changelog} -else - # Merge version from changelog with passed in version. Any zeros in package_version - # will be replaced by corresponding number from changelog version. - base_version_parts=(${latest_version_in_debian_changelog//./ }) - result_version="" - i=0 - for part in $(echo $package_version | tr "." "\n"); do - if [ ${#result_version} -gt 0 ]; then - result_version="${result_version}." - fi - if [ "${part:0:1}" = "0" ]; then - result_version="${result_version}${base_version_parts[$i]}${part:1}" - else - result_version="${result_version}$part" - fi - i=$[i+1] - done - base_version=$result_version -fi -nightlydelimeter=${nightlydelimeter:-"."} -nightlyversion="${base_version}${nightlydelimeter}nightly${timestamp}.${hash_of_current_commit}${package_version_extension}" - -distribution_of_last_change=$(dpkg-parsechangelog --show-field=Distribution) - -if [[ "$DISTS_TO_PROCESS" != *$distribution_of_last_change* ]]; then - distribution_of_last_change=$(echo "$DISTRIBUTIONS_TO_PACKAGE" | cut -d' ' -f1) -fi -changelog_message="Built from commit $hash_of_current_commit" -if [ -n "$most_recent_tag" ]; then - changelog_message="See git log $most_recent_tag..$hash_of_current_commit" -fi - -if [ -z "$preserve_changelog_arg" ]; then - log "Using package version: $nightlyversion-1 for $distribution_of_last_change" - native_version=$nightlyversion -elif [ -n "${package_version}" ]; then - log "Using package version: $base_version-1 for $distribution_of_last_change" - native_version=$base_version -else - log "Using package version: $latest_version_in_debian_changelog" - native_version=$latest_version_in_debian_changelog -fi - -if [ -x "build/buildupdate.mono.sh" ]; then - log "Getting artifacts from TeamCity:" - TRACE ./build/buildupdate.mono.sh -fi - -log "Building source package:" -[ -z "$build_in_place" ] && cd $repo_base_dir/${source_package_dir:-$source_package_name} -if [ -n "$debkeyid" ]; then - signing_opt="-k$debkeyid" - echo "Setting signing opts to ^$signing_opt^" -elif [ -n "$DEBSIGNKEY" ]; then - signing_opt="-k$DEBSIGNKEY" - echo "Setting signing opts to ^$signing_opt^" -fi - -if [ "$(cat debian/source/format)" == "3.0 (native)" ]; then - log "Native package. Building original source package." - # Unfortunately we have to list the files to ignore explicitly and can't use the default - # ignores because we want *.so files in the source package (needed for Mercurial) - TRACE dpkg-source -Zgzip -z6 --tar-ignore=*.a --tar-ignore=*.la --tar-ignore=*.o --tar-ignore=.*.sw? \ - --tar-ignore=*/*~ --tar-ignore=,,* --tar-ignore=.[#~]* --tar-ignore=.deps \ - --tar-ignore=.git --tar-ignore=.gitattributes --tar-ignore=.gitignore \ - --tar-ignore=.gitmodules --tar-ignore=.gitreview --tar-ignore=.hg --tar-ignore=.hgignore \ - --tar-ignore=.hgsigs --tar-ignore=.hgtags --tar-ignore=.svn --tar-ignore=debian -b . - - TRACE mv ../${source_package_name}_${latest_version_in_debian_changelog}.tar.gz ../${source_package_name}_${native_version}.orig.tar.gz - TRACE rm ../${source_package_name}_${latest_version_in_debian_changelog}.dsc - echo "3.0 (quilt)" > debian/source/format - debuild_opts="--source-option=--include-binaries --source-option=--tar-ignore" -fi - -if [ -z "$preserve_changelog_arg" -o -n "${package_version}" ]; then - # Don't change distribution here - we don't want to create a new changelog entry! - dch $NEWPACKAGENAME --newversion "${native_version}-1" --force-bad-version --nomultimaint "$changelog_message" -fi - -TRACE debuild -S -sa -Zgzip -z6 -d ${signing_opt} ${debuild_opts} - -log "Source package files exist with the following sha256sums:" -if [ -z "$build_in_place" ]; then - cd "$repo_base_dir" -else - cd .. -fi -pwd; sha256sum ${source_package_name}_*.{dsc,build,changes,tar.?z} || true diff --git a/docker/scripts/build-and-test.sh b/docker/scripts/build-and-test.sh index f3f89528..f8a3e5e6 100755 --- a/docker/scripts/build-and-test.sh +++ b/docker/scripts/build-and-test.sh @@ -10,32 +10,29 @@ sudo mkdir -p /usr/lib/lfmerge/${DbVersion} echo Running as $(id) # Assuming script is being run from inside the repo, find the repo root and use that as the working directory from now on -echo "Script dir is ${SCRIPT_DIR}" -ls -ld "${SCRIPT_DIR}" cd "${SCRIPT_DIR}" REPO_ROOT="$(git rev-parse --show-toplevel)" -echo "Repo root is ${REPO_ROOT}" -ls -ld "${REPO_ROOT}" cd "${REPO_ROOT}" -"$SCRIPT_DIR"/setup-workspace.sh "${HOME}/packages/lfmerge" +echo "Building packages for version ${DebPackageVersion}" -echo After setup-workspace.sh, pwd is $(pwd) -echo cd to "${HOME}/packages/lfmerge" -cd "${HOME}/packages/lfmerge" +# Explicit restore step so we can save time in later build steps by using --no-restore +echo "Downloading dependencies" +dotnet restore -v:m -"$SCRIPT_DIR"/gitversion-combined.sh ${DbVersion} - -"$SCRIPT_DIR"/download-dependencies-combined.sh ${DbVersion} - -"$SCRIPT_DIR"/compile-lfmerge-combined.sh ${DbVersion} +echo "Compiling LfMerge" +dotnet build --no-restore /v:m /property:Configuration=Release /property:DatabaseVersion=${DbVersion} LfMerge.sln if [ -n "$RUN_UNIT_TESTS" -a "$RUN_UNIT_TESTS" -ne 0 ]; then - "$SCRIPT_DIR"/test-lfmerge-combined.sh ${DbVersion} + echo "Running unit tests" + # dotnet test defaults to killing test processes after 100ms, which is way too short + export VSTEST_TESTHOST_SHUTDOWN_TIMEOUT=30000 # 30 seconds, please, since some of our tests can run very long + # Treat TEST_SPEC enviornment variable as a "contains" operation + if [ -n "$TEST_SPEC" ]; then + dotnet test --no-restore -c Release --filter "FullyQualifiedName~${TEST_SPEC}" + else + dotnet test --no-restore -c Release + fi fi -rm -rf "$SCRIPT_DIR"/data/php # So it doesn't get into the .deb source package - -# "$SCRIPT_DIR"/build-debpackages-combined.sh ${DbVersion} - "$SCRIPT_DIR"/create-installation-tarball.sh ${DbVersion} diff --git a/docker/scripts/build-debpackages-combined.sh b/docker/scripts/build-debpackages-combined.sh deleted file mode 100755 index c1e066f0..00000000 --- a/docker/scripts/build-debpackages-combined.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -e - -echo -e "\033[0;34mBuilding packages for version ${DebPackageVersion} (inserted as ${Version} in .NET AssemblyInfo files)\033[0m" - -#DistributionsToPackage="xenial bionic" -DistributionsToPackage="bionic" - -DEBSIGNKEY=BB89B185D63A1DD5 - -# Needed in setup.sh from Debian packaging scripts. TODO: Investigate why this environment variable is being removed, and at what point -export USER=root - -TRACE() -{ - echo "$@" - "$@" -} - -curDbVersion=$1 - -cd ${HOME}/packages/lfmerge - -mkdir -p finalresults -rm -f finalresults/* -rm -f lfmerge-* -export MONO_PREFIX=/opt/mono5-sil -RUNMODE="PACKAGEBUILD" BUILD=Release . environ - -# for ((curDbVersion=7000068; curDbVersion<=7000070; curDbVersion++)); do - echo -e "\033[0;34mBuilding package for database version ${curDbVersion}\033[0m" - - echo -e "\033[0;34mPrepare source\033[0m" - git clean -dxf --exclude=packages/ --exclude=lib/ - git reset --hard - - if [ -n "$UPDATE_ASSEMBLYINFO_BY_SCRIPT" -a "$UPDATE_ASSEMBLYINFO_BY_SCRIPT" -ne 0 ]; then - find src -name AssemblyInfo.cs -path '*LfMerge*' -print0 | xargs -0 -n 1 ${HOME}/packages/lfmerge/docker/scripts/update-assemblyinfo.sh - fi - - cat > NuGet.Config < - - - - - -EOF - - TRACE /opt/mono5-sil/bin/msbuild /t:PrepareSource /v:detailed build/LfMerge.proj - - TRACE debian/PrepareSource $curDbVersion - - echo -e "\033[0;34mBuild source package\033[0m" - TRACE $HOME/ci-builder-scripts/bash/make-source --dists "$DistributionsToPackage" \ - --arches "amd64" --main-package-name "lfmerge" --source-code-subdir "." \ - --supported-distros "xenial bionic" --debkeyid $DEBSIGNKEY \ - --package-version "$DebPackageVersion" --preserve-changelog - - # echo -e "\033[0;34mBuild binary package\033[0m" - # TRACE $HOME/ci-builder-scripts/bash/build-package --dists "$DistributionsToPackage" \ - # --arches "amd64" --main-package-name "lfmerge" \ - # --build-in-place --supported-distros "xenial bionic" --debkeyid $DEBSIGNKEY --no-upload - - # cd - - # mv results/* finalresults/ - pwd - ls -l .. - mkdir -p finalresults/ - mv ../lfmerge-${curDbVersion}* finalresults/ -# done -ls -lR finalresults - -# Now build binaries right here -cd finalresults -echo Building packages from .dsc file: lfmerge*dsc -DSC=$(ls -1 lfmerge*dsc) -SOURCE=$(grep '^Source: ' "${DSC}" | cut -c9-) -PKGVERSION=$(grep '^Version: ' "${DSC}" | cut -c10- | sed -e 's/-[0-9]\+$//') -# Build dependency pseudo-package, install it, and immediately delete the file, in one command -sudo mk-build-deps -r -i $DSC -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -# Extract source -dpkg-source -x $DSC -ls -l -echo Should be a directory named "${SOURCE}-${PKGVERSION}" -cd "${SOURCE}-${PKGVERSION}" -# Build package, preserving env vars used in MsBuild packaging -debuild -rfakeroot --preserve-envvar Version --preserve-envvar AssemblyVersion --preserve-envvar FileVersion --preserve-envvar InformationalVersion -# Built packages are placed in parent directory, so now we'll delete the unpacked directory so `docker container cp` doesn't collect it -cd .. -rm -rf "${SOURCE}-${PKGVERSION}" diff --git a/docker/scripts/compile-lfmerge-combined.sh b/docker/scripts/compile-lfmerge-combined.sh deleted file mode 100755 index 23cbce95..00000000 --- a/docker/scripts/compile-lfmerge-combined.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -e - -set -e - -echo "Compiling LfMerge and running unit tests" -BUILD=Release . environ - -echo "Using $(which dotnet)" - -export DbVersion="${1-7000072}" -echo "Building for ${DbVersion}" - -dotnet build /t:CompileOnly /v:quiet /property:Configuration=Release /property:DatabaseVersion=${DbVersion} build/LfMerge.proj - -# ln -sf ../Mercurial output/ -# xbuild /t:TestOnly /v:detailed /property:Configuration=Release /property:DatabaseVersion=${DbVersion} build/LfMerge.proj diff --git a/docker/scripts/create-installation-tarball.sh b/docker/scripts/create-installation-tarball.sh index 377d759d..ff29f99f 100755 --- a/docker/scripts/create-installation-tarball.sh +++ b/docker/scripts/create-installation-tarball.sh @@ -14,15 +14,25 @@ export NETSTANDARD=netstandard2.0 export DatabaseVersion=${1:-7000072} # Model version dependent DESTDIR -export DBDESTDIR=tarball/lfmerge-${DatabaseVersion} +export DESTROOT=tarball +export DBDESTDIR=${DESTROOT}/lfmerge-${DatabaseVersion} # Common DESTDIR -export COMMONDESTDIR=tarball/lfmerge +export COMMONDESTDIR=${DESTROOT}/lfmerge export LIB=usr/lib/lfmerge/${DatabaseVersion} export SHARE=usr/share/lfmerge/${DatabaseVersion} export NATIVERUNTIME=runtimes/linux-x64/native export LIBRUNTIME=runtimes/linux-x64/lib export DBVERSIONPATH=/usr/lib/lfmerge/${DatabaseVersion} +# Sanity check before doing anything destructive +if [ -d output/${BUILD}/${FRAMEWORK}/ -a -e output/${BUILD}/${FRAMEWORK}/LfMerge.Core.dll ] +then + echo "Copying installation files into ${DESTROOT}" +else + echo "Build appears to have failed; not touching ${DESTROOT}" + exit 1 +fi + # Apparently the downloaded mercurial.ini doesn't have the right fixutf8 config, and it also # has wrong line endings, so we re-create the entire file cat >Mercurial/mercurial.ini </dev/n install -m 755 output/${BUILD}/${FRAMEWORK}/LfMergeQueueManager ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/LfMergeQueueManager ${DBDESTDIR}/${LIB} install -m 755 output/${BUILD}/${FRAMEWORK}/chorusmerge ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/chorusmerge ${DBDESTDIR}/${LIB} install -m 755 output/${BUILD}/${FRAMEWORK}/FixFwData ${DBDESTDIR}/${LIB} 2>/dev/null || install -m 755 output/${BUILD}/FixFwData ${DBDESTDIR}/${LIB} -(cd ${DBDESTDIR}/${LIB}; ln FixFwData FixFwData.exe) install -d ${DBDESTDIR}/${LIB}/Mercurial install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext install -d ${DBDESTDIR}/${LIB}/Mercurial/hgext/convert diff --git a/docker/scripts/download-dependencies-combined.sh b/docker/scripts/download-dependencies-combined.sh deleted file mode 100755 index 3c9fd147..00000000 --- a/docker/scripts/download-dependencies-combined.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -e - -set -e - -echo "Downloading dependencies" -export MONO_PREFIX=/opt/mono5-sil -. environ - -pwd - -if [ "x$1" = "x7000072" ]; then -msbuild /t:RestorePackages /p:KeepJobsFile=false build/LfMerge.proj -else -msbuild /t:DownloadDependencies /p:KeepJobsFile=false build/LfMerge.proj -fi - diff --git a/docker/scripts/get-version-number.sh b/docker/scripts/get-version-number.sh index dd15e22d..5b682fe8 100755 --- a/docker/scripts/get-version-number.sh +++ b/docker/scripts/get-version-number.sh @@ -36,15 +36,15 @@ fi echo Will calculate version from "${RESULT}" and "${MAJOR}.${MINOR}.${PATCH} with $COMMIT_COUNT commits since then, and current hash $COMMIT_HASH" case "$REV" in - refs/heads/master | refs/heads/fieldworks8-master) + refs/heads/master) PRERELEASE="~alpha.${BUILD_NUMBER}" ;; - refs/heads/qa | refs/heads/fieldworks8-qa) + refs/heads/qa) PRERELEASE="~beta.${BUILD_NUMBER}" ;; - refs/heads/live | refs/heads/fieldworks8-live) + refs/heads/live) PRERELEASE= ;; diff --git a/docker/scripts/gitversion-combined.sh b/docker/scripts/gitversion-combined.sh deleted file mode 100755 index 7a60259a..00000000 --- a/docker/scripts/gitversion-combined.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -e - -SCRIPT_DIR=$(dirname $(readlink -f "$0")) - -echo "Calculated versions:" -echo "DebPackageVersion=${DebPackageVersion}" -echo "MajorMinorPatch=${MajorMinorPatch}" -echo "AssemblyVersion=${AssemblyVersion}" -echo "FileVersion=${FileVersion}" -echo "InformationalVersion=${InformationalVersion}" - -# Fetch tags for GitVersion -git fetch --tags - -# Fetch master for GitVersion -if [ -n "$GITHUB_REF" -a "x$GITHUB_REF" != "xrefs/heads/master" ]; then - git branch --create-reflog master origin/master -fi - -export MONO_PREFIX=/opt/mono5-sil -RUNMODE="PACKAGEBUILD" BUILD=Release . environ -# msbuild build/LfMerge.proj -mkdir -p output/Release - -if [ -n "$UPDATE_ASSEMBLYINFO_BY_SCRIPT" -a "$UPDATE_ASSEMBLYINFO_BY_SCRIPT" -ne 0 ]; then - UPDATE_SCRIPT="${SCRIPT_DIR}/update-assemblyinfo.sh" - echo "Will run ${UPDATE_SCRIPT} to update AssemblyInfo.cs files" - [ -x "${UPDATE_SCRIPT}" ] && find src -name AssemblyInfo.cs -path '*LfMerge*' -print0 | xargs -0 -n 1 "${UPDATE_SCRIPT}" -fi - -echo "Building packages for version ${DebPackageVersion}" diff --git a/docker/scripts/setup-workspace.sh b/docker/scripts/setup-workspace.sh deleted file mode 100755 index 9edc84af..00000000 --- a/docker/scripts/setup-workspace.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -set -e - -DEST="${1:-${HOME}/packages/lfmerge}" - -export MONO_PREFIX=/opt/mono5-sil - -mkdir -p "${HOME}/.gnupg" "${HOME}/ci-builder-scripts/bash" "${DEST}" - -REPO_ROOT="$(git rev-parse --show-toplevel)" - -# cp -a "${REPO_ROOT}" "${DEST}" creates ${DEST}/repo and then everything is under there. That's not actually what we want. So... -sudo cp -a "${REPO_ROOT}"/* "${REPO_ROOT}"/.[a-zA-Z0-9]* "${DEST}" -sudo chown -R builder:users "${DEST}" -# The make-source shell script (and its common.sh helper) expects to live under ${HOME}/ci-builder-scripts/bash, so make sure that's the case -mkdir -p "${HOME}/ci-builder-scripts/bash" -cp "${DEST}/docker/common.sh" "${HOME}/ci-builder-scripts/bash/" -cp "${DEST}/docker/make-source" "${HOME}/ci-builder-scripts/bash/" - -cd "${DEST}" -if [ "${BRANCH_TO_BUILD}" ]; then - git checkout "${BRANCH_TO_BUILD}" -fi - -git clean -dxf --exclude=packages/ --exclude=build/packages/ -git reset --hard - -# FLExBridge dependencies from FW 8 builds have vanished from TeamCity, so we stored them in the Docker image under ${REPO_ROOT}/docker -mkdir -p lib -(cd lib && xz -dc "${REPO_ROOT}/docker/fw8-flexbridge.tar.xz" | tar xf -) diff --git a/docker/scripts/test-lfmerge-combined.sh b/docker/scripts/test-lfmerge-combined.sh deleted file mode 100755 index 2b2d1dc0..00000000 --- a/docker/scripts/test-lfmerge-combined.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -e - -export DbVersion="${1-7000072}" - -echo "Running unit tests for ${DbVersion}" -BUILD=Release . environ - -echo "Using $(which mono)" -export FrameworkPathOverride=/opt/mono5-sil/lib/mono/4.5 - -ln -sf ../Mercurial output/ -if [ -n "$TEST_SPEC" ]; then - echo Only running "$TEST_SPEC" - if [ "x$NUNIT_VERSION_MAJOR" = "x2" ]; then - mono packages/NUnit.Runners.Net4.2.6.4/tools/nunit-console.exe output/Release/LfMerge*.Tests.dll -run "$TEST_SPEC" - elif [ "x$NUNIT_VERSION_MAJOR" = "x3" ]; then - mono packages/NUnit.ConsoleRunner/tools/nunit3-console.exe output/Release/LfMerge*.Tests.dll --test "$TEST_SPEC" - else - echo Warning, NUnit version not specified. Assuming NUnit 3. - mono packages/NUnit.ConsoleRunner/tools/nunit3-console.exe output/Release/LfMerge*.Tests.dll --test "$TEST_SPEC" - fi -else - msbuild /t:TestOnly /v:detailed /property:Configuration=Release build/LfMerge.proj -fi diff --git a/docker/scripts/update-assemblyinfo.sh b/docker/scripts/update-assemblyinfo.sh deleted file mode 100755 index 832b53c9..00000000 --- a/docker/scripts/update-assemblyinfo.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "Error: no filename given. Please pass a filename on the command line." - exit 1 -fi -if [ $# -gt 1 ]; then - echo "Warning: multiple filenames passed. Only $1 will be processed." >&2 -fi - -# Rules for auto-generated AssemblyInfo in .Net Core 3.1 or later, which we will replicate: -# AssemblyVersion and FileVersion default to the value of $(Version) without the suffix. For example, if $(Version) is 1.2.3-beta.4, then the value would be 1.2.3. -# InformationalVersion defaults to the value of $(Version). -# Source: https://docs.microsoft.com/en-us/dotnet/core/project-sdk/msbuild-props#generateassemblyinfo - -Version=${Version:-0.0.1} - -InformationalVersion=${InformationalVersion:-$Version} -AssemblyVersion=${AssemblyVersion:-$(echo "$Version" | sed -E 's/^([^-]*)+.*/\1/')} -FileVersion=${FileVersion:-$(echo "$Version" | sed -E 's/^([^-]*)+.*/\1/')} - -# Assembly attribute lines look like this: -# [assembly: AssemblyVersion("1.2.3.4")] - -tmpfname="$(mktemp)" -fname="$1" - -echo "Updating ${fname} with AssemblyVersion ${AssemblyVersion} and InformationalVersion ${InformationalVersion}" - -# First remove any existing lines -cat "$fname" \ -| sed '/^\[assembly: AssemblyVersion("[^"]*")\]$/d' \ -| sed '/^\[assembly: AssemblyFileVersion("[^"]*")\]$/d' \ -| sed '/^\[assembly: AssemblyInformationalVersion("[^"]*")\]$/d' \ -> "$tmpfname" - -# Then append new lines at the end -echo "[assembly: AssemblyVersion(\"${AssemblyVersion}\")]" >> "$tmpfname" -echo "[assembly: AssemblyFileVersion(\"${FileVersion}\")]" >> "$tmpfname" -echo "[assembly: AssemblyInformationalVersion(\"${InformationalVersion}\")]" >> "$tmpfname" - -mv "${tmpfname}" "${fname}" diff --git a/docker/sil-packages-key.gpg b/docker/sil-packages-key.gpg deleted file mode 100644 index 316085a7..00000000 --- a/docker/sil-packages-key.gpg +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBFhFlhUBEADQJ7pmIGkKNbju529klmZn7BmngvLFlRlPRAgBF8nx29pj56+s -+PFIvN/fionvisnnTqHmPoDCpfxSTyso4VHkV1fvYpB1vthdl4xRlZrdq+1zaghz -yeHj5T8hnWwwdP/na7MCEFv1qnPVWOPXrLH2WFjrmDDvZrzcY6XK/5MYx/E40mbL -4d5yp/4JRp6BX2eRIJaAw33EKZ3799xgSQzbAwtf4LZzZoBxFUW+fzFVrGShlS5O -eODHD4mtWKrd1rZeri/6FV7xGR7CU2F3rnDpByP5MtWp0no/RQWX2PO9PwI5WZAz -WopDGt+/IBaJUW0zc4F+8bRQjHKu3dnyVf5mZ1KhxMGilKqA0gyabSPSLBw4WRDb -+GvenzZoCNmVBf69vJkiyzB+4ADmZsx+v7coOa7kvlinxW04tlDTfzH2hrp0xNcP -Whb0VFkYSxmavkgOegMvOPFksFY6QRu0VQt1u8Zhpo1LMe/Sk+ORdeby1EzhyINM -jzM6u32HxjGdnJqWTUTLi1IrQgLnlM//QTtl5xUVZMvcrQgBCFfVfp8D+DPR7n2u -q/ec2xDg7fBPSdvhBLKgXKJR/yS9L7BFP8jEXiHTUCiHickNN27vB2TvaLDkpBav -iW8xjKBb13RbwFxEFvNJPjMH2czgyqfnO7DGvCNl7aMgrH9ZuO41WJyjRQARAQAB -tCdTSUwgUGFja2FnZXMgPGFyY2hpdmVAcGFja2FnZXMuc2lsLm9yZz6JAjcEEwEI -ACEFAlhFlhUCGwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQ7NMEvWVSO7nN -XA//TIUre4VhBbltSfPmtNGm+dMgEwF4wW9KBqY8qrcUdob3qCd80Tbz3ZwDG6J0 -oNh+UP5Bx3d/jg4GjYynTw+v5Y4B0SK1Vmf6yXZc+YVN3KJKZKM9GFbolQeK6Kov -4BgcdElw4NeiveUKeBLzmxg+b1MZwpcnVIThDY4n4jyhRPjS3LEgbZzUUbTB1d+f -Goggh544scjBUAnuFoMeYKldyN3mHOZAtKTgMfeajwqB9U2CU5AcL9cmOzTH6Iuv -ZjJdKyXSJf5B5tFAoXhz4zXedpYxvDUvE/evVYAr2Qvj1+/2v7BroEEqmBGDeUlp -1QW/pXBN8JmKXgamEh8sY7EVxRx3+HPlRN0Pc4iPqc8SjpuZwA7URj0ooHmNYm/+ -Cdno71/5vs833WgLGspBVFdAOREPabIBzTFbT2HW+hBvArcVlZXpLsArbd9wwbY1 -n/6MP1kPtAnVtA8+oDo+BpRTZINaYOsy0X+8TQ+j/RVBaIFJT6kSbjxxAZHTV7sS -1vHzSVKW+Pz9kCb+trFnlll0zuLvztzL/u569Duv3Q7SrxXjr+9oOMiZsqb56hBB -ddvDhvoOhZg/B74nRz5pdXl3GK0oY5b5BI47hKX2sCtI/s+V8jM/DuyHt1vH+oaL -OxRe+GqHgM3HVqG/mlRWfCHqvhdDERjTm/D47Agi7xiEL6w= -=DSkQ ------END PGP PUBLIC KEY BLOCK----- diff --git a/docker/sil-packages-testing-key.gpg b/docker/sil-packages-testing-key.gpg deleted file mode 100644 index cf38fe83..00000000 --- a/docker/sil-packages-testing-key.gpg +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQENBFILpkEBCACtt6o/TNmy4jb0buz5/ex9TW2BgP3yCRwgt5BbSwWJ9OWqn+Kr -YTyCZuJFAjjeNwnkOs4rR7k83wkIUMe+FHa569i15QXiqPUbBx8xqQ44KmzIX3gl -J2sKpPU3tRIJyAGQebQQFgTYTT7ZjMFDNniWkfhrkXK2Uyj786AFcCIcM4ahVBfJ -PrjQzrbl0zgY6tG40PDfy+zjUzICNZSoJtzTCEvMy4g+XCy8QMobtNaH63Q+uEdF -cpA/izvZedZGNcF5Mo+pENuFO9Rlq5/A1VTlor0oyG247oY65k/YLS7IktrZsS5L -SeOdSKSy7ZgADbZUYEasAyqAyh3zWmgcwuvbABEBAAG0PFNJTCBQYWNrYWdlcyAo -VGVzdGluZyBSZXBvc2l0b3J5KSA8YXJjaGl2ZUBwYWNrYWdlcy5zaWwub3JnPokB -OAQTAQIAIgUCUgumQQIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQoiYD -oOZvjb0wRwf/YX10v55uzp0P07ntLrBd02k1BdmKNZcrrbN7Bw+YkAuOOLkk4c9a -iPP967F1bMxWX8DZGMZQnJkM+5wYyEUmR59823R1cy02sxvhQZEy+rEloomJomw2 -7dPON0bG4ZkmbZpMvgREWgQf2JGpCiH0UkIQyHjnYIJ3Jb5U5sZAttDKNW6pF1U4 -pQu1TFq0DFfaWbf+XMYUDhYPcsUBlFaTiQlLZW7tJgGJt3HAby/2lyRTcvXR6p4E -7gnm9N4wW13AFx7T2xzXdtVS5JmXY51qSUSgNmgRQg9Nqxs1r8KJaEYiM7wf0/We -PlqvkAoU/uIUGoTR0S2kpmvfMC3hy954d7kBDQRSC6ZBAQgAujEqmBB2L74H4ZzS -EQNDZbu4ZgrV7ax/8SmjKP0EYzfcw2/egjTydE80DzAkLaGqHoHCvvYK3OGyBVZm -tDFjepAy5+6axPReRiwQn2GPqEvy++zx0NWHstm2qMp6xxhXYctmPCatD7U/j4J4 -3fuhd0N0sVJhve6XuPLe1q9HWOOn+xlxHmUgW3nnICXAKuOW5RQqyLID4Wczg/g/ -TXzrKsBKSCLeuPa59zORbHszGbK3a301o0zw+5GTmAtbDdIPJEdRlcs3tan8fHIa -y19T16HCF70L5N8lQPfz7zRvMK6R25Dnax4JfoHhpyMXpS1cuyeq46gz2mWCrkCq -VbpZVQARAQABiQEfBBgBAgAJBQJSC6ZBAhsMAAoJEKImA6Dmb429xwIH/jtEVvuP -yVTFZwN/qKCeSA81PEXvZLtHH9mIxD2t5xHgkTHmxeFERxStOHljFiLhL8bHy1Io -BVaqEQd3QrEoMP2Ebgwdq4Lq2ZQwy3xjuX3r1EdHci5Vu8NYuSAXUqKRiO0yE9St -Yc1Pnrs9npjuUI+jFQa5LRf9l5gQAM6OcHzvVe/tNyht0GWAj1iTaFW8eKMub6BX -GRaXAYeR8nKuFd5jIX7yvJzo3gIYGb924G285BjcHCiqGmuAxGinw0/OolvTbaUd -xaz8230EF5P6UD1yE/pbaXTwOFBqJoF9qKvC1Y159s9BOZsqwFQHsQLw/7RZC0J7 -Id1T1+d1fQW7HUc= -=/rsE ------END PGP PUBLIC KEY BLOCK----- diff --git a/entrypoint.sh b/entrypoint.sh index c3dd2c5a..19780923 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,18 +5,6 @@ echo Entrypoint running with args "$@" # Catch SIGTERM and exit cleanly trap "exit" TERM -# rsyslog needs to run so that lfmerge can log to /var/log/syslog -/etc/init.d/rsyslog start - -# Ensure /var/log/syslog exists so tail -f /var/log/syslog will run -logger "Starting container..." -# echo /var/log/syslog to container stdout so it shows up in `kubectl logs` -# First waiting for syslog to exist (race condition: `logger` can return before syslog file is created) -until [ -r /var/log/syslog ]; do - sleep 0.2 -done -tail -f /var/log/syslog & - # run lfmergeqm on startup to clear out any failed send/receive sessions from previous container /lfmergeqm-background.sh & # MUST be run as a background process as it kicks off an infinite loop to run every 24 hours diff --git a/environ b/environ index 5e186b97..fca98dab 100644 --- a/environ +++ b/environ @@ -1,78 +1,10 @@ -# Environment settings for running programs with the SIL version of mono -# Set MONO_ENVIRON to this file's pathname, then run, for example, -# /opt/mono5-sil/bin/mono --debug LfMerge.exe -# These settings assume that the packaged SIL Mono is installed in /opt/mono5-sil. +# A few necessary environment variables for the Mercurial build from Chorus to run correctly # Note that this file is intended to be "sourced", not "executed". # the sourcing script should cd/pushd to the directory containing this script BASE="$(pwd)" -DEFAULT_MONO_PREFIX=/opt/mono5-sil -[ -z "$BUILD" ] && BUILD=Debug -[ -z "$MONO_PREFIX" ] && MONO_PREFIX=${DEFAULT_MONO_PREFIX} -MONO_DEBUG=explicit-null-checks -MONO_ENV_OPTIONS="-O=-gshared" - -if [ "$RUNMODE" != "NUGET" ]; then - [ -z "$DBVERSIONPATH" ] && DBVERSIONPATH=$(find /usr/lib/lfmerge -maxdepth 1 -type d -name [0-9]\* | sort | tail -n 1) - COMPONENTS_MAP_PATH=$DBVERSIONPATH -fi -ICU_DATA=/usr/share/fieldworks/Icu54 - -################################################################################################ - -if [ "$RUNMODE" = "INSTALLED" ] -then - # Add the installation directory to the library paths - LD_LIBRARY_PATH="${BASE}:${LD_LIBRARY_PATH}" - MONO_PATH="${BASE}:${MONO_PATH}" -else - # Add the build output to paths - PATH="${BASE}/output/${BUILD}:${PATH}" - LD_LIBRARY_PATH="${BASE}/output/${BUILD}:${LD_LIBRARY_PATH}" - MONO_PATH="${BASE}/output/${BUILD}:${MONO_PATH}" - [ -f "${BASE}/output/${BUILD}/components.map" ] && COMPONENTS_MAP_PATH="${BASE}/output/${BUILD}" -fi - -################################################################################################ - -if [ "$RUNMODE" = "PACKAGE" -o "$RUNMODE" = "INSTALLED" ] -then - # Add packaged mono items to paths - PATH="${DEFAULT_MONO_PREFIX}/bin:${PATH}" - LD_LIBRARY_PATH="${DEFAULT_MONO_PREFIX}/lib:${LD_LIBRARY_PATH}" - PKG_CONFIG_PATH="${DEFAULT_MONO_PREFIX}/lib/pkgconfig:${PKG_CONFIG_PATH}" - MONO_GAC_PREFIX="${DEFAULT_MONO_PREFIX}:/usr" -else - # Add locally-built mono items to paths - # We also add the default values for PKG_CONFIG_PATH - MonoDevelop resets the PKG_CONFIG_PATH - # environment variable if we're running under a different Mono runtime so that some - # packages that got installed in standard locations can't be found otherwise. - PATH="${MONO_PREFIX}/bin:${PATH}" - LD_LIBRARY_PATH="${MONO_PREFIX}/lib:${LD_LIBRARY_PATH}" - PKG_CONFIG_PATH="${MONO_PREFIX}/lib/pkgconfig:${PKG_CONFIG_PATH}:/usr/local/lib/pkgconfig:/lib/pkgconfig:/usr/lib/pkgconfig" - MONO_GAC_PREFIX="${MONO_PREFIX}:/usr" -fi - -################################################################################################ - -LD_LIBRARY_PATH="${DBVERSIONPATH}:${LD_LIBRARY_PATH}" - -FW_ROOTCODE=/usr/share/fieldworks - -MONO_TRACE_LISTENER="Console.Out" - -# if debugging for performance unset DEBUG_ENABLE_PTR_VALIDATION env var. -#export DEBUG_ENABLE_PTR_VALIDATION=1 - -export \ - PATH LD_LIBRARY_PATH PKG_CONFIG_PATH LD_PRELOAD \ - COMPONENTS_MAP_PATH ICU_DATA \ - MONO_PATH \ - MONO_PREFIX MONO_GAC_PREFIX \ - MONO_TRACE_LISTENER \ - MONO_DEBUG MONO_ENV_OPTIONS \ - FW_ROOTCODE +export PATH="${BASE}/output/${BUILD}:${BASE}/output/Mercurial:${PATH}" # set HGRCPATH so that we ignore ~/.hgrc files which might have content that is # incompatible with our version of Mercurial diff --git a/lfmerge.conf b/lfmerge.conf deleted file mode 100644 index 45715cd6..00000000 --- a/lfmerge.conf +++ /dev/null @@ -1,10 +0,0 @@ -# Configuration file for LfMerge -BaseDir = /var/lib/languageforge/lexicon/sendreceive -WebworkDir = webwork -TemplatesDir = Templates -MongoHostname = db -MongoPort = 27017 -MongoMainDatabaseName = scriptureforge -MongoDatabaseNamePrefix = sf_ -VerboseProgress = true -PhpSourcePath = /var/www/html diff --git a/pbuild.sh b/pbuild.sh index 027b2d06..19983576 100755 --- a/pbuild.sh +++ b/pbuild.sh @@ -1,124 +1,81 @@ #!/bin/bash -which parallel >/dev/null -if [ $? -ne 0 ]; then - echo 'Please run "sudo apt-get install parallel" and try again.' - exit 1 -fi - -mkdir -p /storage/nuget -if [ $? -ne 0 ]; then - echo "Please create a /storage directory and then run 'chown ${USER} /storage', to be able to cache NuGet packages" - exit 1 -fi -# TODO: Check for rwxrwsr-x permissions and appropriate uid/gid settings +set -e + +# These are arrays; see https://www.gnu.org/software/bash/manual/html_node/Arrays.html +DBMODEL_VERSIONS=(7000072) +HISTORICAL_VERSIONS=(7000068 7000069 7000070) + +# In the future when we have more than one model version, we may want to use GNU parallel for building. +# ATTENTION: If GNU parallel is desired, uncomment the below (until the "ATTENTION: Stop uncommenting here" line): + +# DBMODEL_COUNT=${#DBMODEL_VERSIONS[@]} +# MULTIPLE_VERSIONS=0 +# if [ $DBMODEL_COUNT -gt 1 ]; then +# MULTIPLE_VERSIONS=1 +# fi + +# # Use GNU parallel only if we have multiple DBVersions to deal with +# # Specify USE_PARALLEL=0 or USE_PARALLEL=1 in environment to override this default +# USE_PARALLEL=${USE_PARALLEL:-$MULTIPLE_VERSIONS} + +# # echo We have ${DBMODEL_COUNT} versions: "${DBMODEL_VERSIONS[@]}" +# if [ $USE_PARALLEL -gt 0 ]; then +# which parallel >/dev/null +# if [ $? -ne 0 ]; then +# echo 'Please run "sudo apt-get install parallel" and try again.' +# exit 1 +# fi +# else +# echo GNU parallel will not be used +# fi + +# ATTENTION: Stop uncommenting here # Find appropriate branch(es) to build -CURRENT_BRANCH="$(git name-rev --name-only HEAD)" -PARENT_MAJOR_VERSION=$(git describe --long --match "v*" | cut -c1-2) -BUILD_FW8=1 - -# FW8 branches will have ancestors tagged v1.x, while FW9 branches will have ancestors tagged v2.x -if [ "x$PARENT_MAJOR_VERSION" = "xv2" ]; then - IS_FW9=true -else - IS_FW9="" -fi - -if [ "${IS_FW9}" ]; then - echo Current branch is FW9, detecting FW8 branch to use... - FW9_BUILD_BRANCH="${CURRENT_BRANCH}" - if [ "$1" ]; then - if [ "$1" = "--no-fw8" ]; then - BUILD_FW8=0 - FW8_BUILD_BRANCH="" - else - FW8_BUILD_BRANCH="$1" - fi - elif [ "${CURRENT_BRANCH}" = "master" -o "${CURRENT_BRANCH}" = "qa" -o "${CURRENT_BRANCH}" = "live" ]; then - FW8_BUILD_BRANCH="fieldworks8-${CURRENT_BRANCH}" - else - echo No FW 8 branch specified, assuming fieldworks8-master - echo To specify a different branch, run pbuild.sh '<'fw8-branch'>', e.g. '"'pbuild.sh feature/some-fw8-branch'"' - echo 'Or to skip FW 8 branch entirely, run pbuild.sh --no-fw8' - FW8_BUILD_BRANCH="fieldworks8-master" - fi -else - echo Current branch is FW8, detecting FW9 branch to use... - FW8_BUILD_BRANCH="${CURRENT_BRANCH}" - if [ "$1" ]; then - FW9_BUILD_BRANCH="$1" - elif [ "${CURRENT_BRANCH}" = "fieldworks8-master" -o "${CURRENT_BRANCH}" = "fieldworks8-qa" -o "${CURRENT_BRANCH}" = "fieldworks8-live" ]; then - FW9_BUILD_BRANCH="${CURRENT_BRANCH##fieldworks8-}" - else - echo No FW 9 branch specified, assuming master - echo To specify a different branch, run pbuild.sh '<'fw8-branch'>', e.g. '"'pbuild.sh feature/some-fw8-branch'"' - FW9_BUILD_BRANCH="master" - fi -fi - -if [ "${BUILD_FW8}" -eq 0 ]; then - echo Will build ONLY the FW9 build, from "${FW9_BUILD_BRANCH}" -else - echo Will build FW9 build from "${FW9_BUILD_BRANCH}" and FW8 builds from "${FW8_BUILD_BRANCH}" -fi +FW9_BUILD_BRANCH="$(git name-rev --name-only HEAD)" +echo Will build ONLY the FW9 build, from "${FW9_BUILD_BRANCH}" # Clean up any previous builds -# This for loop should be across all DbVersions even if BUILD_FW8 is 0 -for f in 68 69 70 72; do - # Can safely ignore "container doesn't exist" as that's not an error - docker container kill tmp-lfmerge-build-70000${f} >/dev/null 2>/dev/null || true - docker container rm tmp-lfmerge-build-70000${f} >/dev/null 2>/dev/null || true +# This for loop includes all historical DbVersions even if BUILD_FW8 is 0 +for DbVersion in ${HISTORICAL_VERSIONS[@]} ${DBMODEL_VERSIONS[@]}; do + # Can safely ignore "container doesn't exist" as that's not an error + docker container kill tmp-lfmerge-build-${DbVersion} >/dev/null 2>/dev/null || true + docker container rm tmp-lfmerge-build-${DbVersion} >/dev/null 2>/dev/null || true done CURRENT_UID=$(id -u) -# First create the base build container ONCE (not in parallel), to ensure that the slow steps (apt-get install mono5-sil) are cached +# First create the base build container ONCE (it will be reused as a base by each DbVersion build), which should help with caching +docker build -t ghcr.io/sillsdev/lfmerge-base:sdk -f Dockerfile.builder-base . +docker build -t ghcr.io/sillsdev/lfmerge-base:runtime -f Dockerfile.runtime-base . docker build --build-arg "BUILDER_UID=${CURRENT_UID}" -t lfmerge-builder-base --target lfmerge-builder-base . -# Create the build images for each DbVersion in parallel -# NOTE: now that the differences are only ENV lines, parallel no longer gains any time. Should we turn this into a for loop? -if [ "${BUILD_FW8}" -eq 0 ]; then - docker build --build-arg DbVersion=7000072 --build-arg "BUILDER_UID=${CURRENT_UID}" -t lfmerge-build-7000072 . -else - time parallel --no-notice < + + + + + diff --git a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj index 4bbca072..614038aa 100644 --- a/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj +++ b/src/LfMerge.Core.Tests/LfMerge.Core.Tests.csproj @@ -33,8 +33,9 @@ See full changelog at https://github.com/sillsdev/LfMerge/blob/master/CHANGELOG. - - + + + diff --git a/src/LfMerge.Core.Tests/MercurialServer.cs b/src/LfMerge.Core.Tests/MercurialServer.cs index c8c0fa03..5573d712 100644 --- a/src/LfMerge.Core.Tests/MercurialServer.cs +++ b/src/LfMerge.Core.Tests/MercurialServer.cs @@ -46,7 +46,7 @@ public void Stop() var pid = File.ReadAllText(_pidFile); if (Platform.IsLinux) { - CommandLineRunner.Run("kill", "-9 " + pid, Directory.GetCurrentDirectory(), + CommandLineRunner.Run("sh", "-c 'kill -9 " + pid + "'", Directory.GetCurrentDirectory(), 120, new NullProgress()); } else diff --git a/src/LfMerge.Core.Tests/ProcessingStateTests.cs b/src/LfMerge.Core.Tests/ProcessingStateTests.cs index 02dee0f6..290ea1bf 100644 --- a/src/LfMerge.Core.Tests/ProcessingStateTests.cs +++ b/src/LfMerge.Core.Tests/ProcessingStateTests.cs @@ -138,6 +138,7 @@ public void State_SettingProperty_SerializesState() ""CurrentStep"": 1, ""RetryCounter"": 2, ""UncommittedEditCounter"": 0, + ""Error"": null, ""ErrorMessage"": null, ""ErrorCode"": 0, ""PreviousRunTotalMilliseconds"": 0, diff --git a/src/LfMerge.Core/LfMerge.Core.csproj b/src/LfMerge.Core/LfMerge.Core.csproj index ef61cfb9..253657e9 100644 --- a/src/LfMerge.Core/LfMerge.Core.csproj +++ b/src/LfMerge.Core/LfMerge.Core.csproj @@ -39,6 +39,7 @@ See full changelog at https://github.com/sillsdev/LfMerge/blob/master/CHANGELOG. + diff --git a/src/LfMerge.Core/Settings/LfMergeSettings.cs b/src/LfMerge.Core/Settings/LfMergeSettings.cs index 6baf6f8d..7df532c4 100644 --- a/src/LfMerge.Core/Settings/LfMergeSettings.cs +++ b/src/LfMerge.Core/Settings/LfMergeSettings.cs @@ -10,121 +10,55 @@ namespace LfMerge.Core.Settings { public class LfMergeSettings { - private string _baseDir = null; - private string _webworkDir = null; - private string _templatesDir = null; - private string _mongoHostname = null; - private int _mongoPort = 0; - private string _mongoMainDatabaseName = null; - private string _mongoDatabaseNamePrefix = null; - private string _verboseProgressStr = null; - private bool _verboseProgress = false; - private string _languageDepotRepoUri = null; - // Settings derived from environment variables - public string BaseDir { - get { - if (_baseDir != null) return _baseDir; - else { - _baseDir = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_BaseDir) ?? DefaultLfMergeSettings.BaseDir; - return _baseDir; - } - } - } + public string BaseDir => Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_BaseDir) ?? DefaultLfMergeSettings.BaseDir; public string WebworkDir { get { - if (_webworkDir != null) return _webworkDir; - else { - _webworkDir = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_WebworkDir) ?? DefaultLfMergeSettings.WebworkDir; - _webworkDir = Path.IsPathRooted(_webworkDir) ? _webworkDir : Path.Combine(BaseDir, _webworkDir); - return _webworkDir; - } + string _webworkDir = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_WebworkDir) ?? DefaultLfMergeSettings.WebworkDir; + _webworkDir = Path.IsPathRooted(_webworkDir) ? _webworkDir : Path.Combine(BaseDir, _webworkDir); + return _webworkDir; } } public string TemplatesDir { get { - if (_templatesDir != null) return _templatesDir; - else { - _templatesDir = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_TemplatesDir) ?? DefaultLfMergeSettings.TemplatesDir; - _templatesDir = Path.IsPathRooted(_templatesDir) ? _templatesDir : Path.Combine(BaseDir, _templatesDir); - return _templatesDir; - } + string _templatesDir = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_TemplatesDir) ?? DefaultLfMergeSettings.TemplatesDir; + _templatesDir = Path.IsPathRooted(_templatesDir) ? _templatesDir : Path.Combine(BaseDir, _templatesDir); + return _templatesDir; } } - public string MongoHostname { - get { - if (_mongoHostname != null) return _mongoHostname; - else { - _mongoHostname = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_MongoHostname) ?? DefaultLfMergeSettings.MongoHostname; - return _mongoHostname; - } - } - } + public string MongoHostname => Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_MongoHostname) ?? DefaultLfMergeSettings.MongoHostname; public int MongoPort { get { - if (_mongoPort != 0) return _mongoPort; - else { - string mongoPortStr = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_MongoPort); - if (mongoPortStr == null) mongoPortStr = ""; - if (Int32.TryParse(mongoPortStr, out _mongoPort)) { - return _mongoPort; - } else { - _mongoPort = DefaultLfMergeSettings.MongoPort; - return _mongoPort; - } + string _mongoPortStr = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_MongoPort); + if (_mongoPortStr == null) _mongoPortStr = ""; + if (Int32.TryParse(_mongoPortStr, out int _mongoPort)) { + return _mongoPort; + } else { + return DefaultLfMergeSettings.MongoPort; } } } - public string MongoMainDatabaseName { - get { - if (_mongoMainDatabaseName != null) return _mongoMainDatabaseName; - else { - _mongoMainDatabaseName = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_MongoMainDatabaseName) ?? DefaultLfMergeSettings.MongoMainDatabaseName; - return _mongoMainDatabaseName; - } - } - } + public string MongoMainDatabaseName => Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_MongoMainDatabaseName) ?? DefaultLfMergeSettings.MongoMainDatabaseName; /// /// The prefix prepended to project codes to get the Mongo database name. /// - public string MongoDatabaseNamePrefix { - get { - if (_mongoDatabaseNamePrefix != null) return _mongoDatabaseNamePrefix; - else { - _mongoDatabaseNamePrefix = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_MongoDatabaseNamePrefix) ?? DefaultLfMergeSettings.MongoDatabaseNamePrefix; - return _mongoDatabaseNamePrefix; - } - } - } + public string MongoDatabaseNamePrefix => Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_MongoDatabaseNamePrefix) ?? DefaultLfMergeSettings.MongoDatabaseNamePrefix; public bool VerboseProgress { get { - if (_verboseProgressStr != null) return _verboseProgress; - else { - _verboseProgressStr = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_VerboseProgress); - if (_verboseProgressStr == null) _verboseProgressStr = ""; - _verboseProgress = LanguageForge.Model.ParseBoolean.FromString(_verboseProgressStr); - return _verboseProgress; - } + string _verboseProgressStr = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_VerboseProgress) ?? ""; + return LanguageForge.Model.ParseBoolean.FromString(_verboseProgressStr); } } - public string LanguageDepotRepoUri { - get { - if (_languageDepotRepoUri != null) return _languageDepotRepoUri; - else { - _languageDepotRepoUri = Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri) ?? DefaultLfMergeSettings.LanguageDepotRepoUri; - return _languageDepotRepoUri; - } - } - } + public string LanguageDepotRepoUri => Environment.GetEnvironmentVariable(MagicStrings.SettingsEnvVar_LanguageDepotRepoUri) ?? DefaultLfMergeSettings.LanguageDepotRepoUri; // Settings calculated at runtime from sources other than environment variables @@ -150,6 +84,7 @@ public LfMergeSettings() public void Initialize() { // TODO: Get rid of this once we simplify the queue system. 2022-02 RM + SetAllMembers(); Queue.CreateQueueDirectories(this); } diff --git a/src/LfMerge.Tests/LfMerge.Tests.csproj b/src/LfMerge.Tests/LfMerge.Tests.csproj index 81c6a4a0..565106f8 100644 --- a/src/LfMerge.Tests/LfMerge.Tests.csproj +++ b/src/LfMerge.Tests/LfMerge.Tests.csproj @@ -33,8 +33,9 @@ See full changelog at https://github.com/sillsdev/LfMerge/blob/master/CHANGELOG. - + +