From 7a962e58962723aa238688711c2332f772168440 Mon Sep 17 00:00:00 2001
From: Marshall Ward <marshall.ward@noaa.gov>
Date: Wed, 23 Nov 2022 17:40:54 -0500
Subject: [PATCH] .testing: Fix concurrency errors in tc4 rules

This patch fixes two issues in the preprocessing of tc4.

* ocean_hgrid.nc is marked as a dependency of gen_data
* Multiple ouputs are handled more safely in the Makefile

Expanding on the second point: We were directing one rule to produce two
output files, which resulted in the rule being run twice when invoked in
parallel (make -j).

This has been replaced with the recommended solution for handling
concurrent outputs: use one to generate both, and connect the second to
the first with a separate rule.

I have also generalized the `make` command in the .testing Makefile.

This should address (and hopefully fix) some intermittent errors in the
.testing build on Gaea.
---
 .testing/Makefile        |  2 +-
 .testing/tc4/Makefile.in | 33 +++++++++++++++++++++++++++------
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/.testing/Makefile b/.testing/Makefile
index 3e5c174239..73a97229d4 100644
--- a/.testing/Makefile
+++ b/.testing/Makefile
@@ -522,7 +522,7 @@ $(foreach c,$(CONFIGS),$(eval $(call CONFIG_DIM_RULE,$(c))))
 # NOTE: This only support tc4, but can be generalized over all tests.
 .PHONY: preproc
 preproc: tc4/Makefile
-	cd tc4 && make
+	cd tc4 && $(MAKE) LAUNCHER="$(MPIRUN)"
 
 tc4/Makefile: tc4/configure tc4/Makefile.in
 	cd $(@D) && ./configure || (cat config.log && false)
diff --git a/.testing/tc4/Makefile.in b/.testing/tc4/Makefile.in
index 249d86b0b6..5a0e441482 100644
--- a/.testing/tc4/Makefile.in
+++ b/.testing/tc4/Makefile.in
@@ -4,15 +4,33 @@ FCFLAGS = @FCFLAGS@
 LDFLAGS = @LDFLAGS@
 LIBS = @LIBS@
 
-OUT = topog.nc ocean_hgrid.nc temp_salt_ic.nc sponge.nc
+LAUNCHER ?=
 
-all: $(OUT)
+OUT = ocean_hgrid.nc topog.nc temp_salt_ic.nc sponge.nc
 
-ocean_hgrid.nc topog.nc: gen_grid
-	./gen_grid
+# Since each program generates two outputs, we can only use one to track the
+# creation.  The second rule is used to indirectly re-invoke the first rule.
+#
+# Reference:
+# https://www.gnu.org/software/automake/manual/html_node/Multiple-Outputs.html
 
-temp_salt_ic.nc sponge.nc: gen_data
-	./gen_data
+# Program output
+all: ocean_hgrid.nc temp_salt_ic.nc
+
+ocean_hgrid.nc: gen_grid
+	$(LAUNCHER) ./gen_grid
+topog.nc: ocean_hgrid.nc
+	@test -f $@ || rm -f $^
+	@test -f $@ || $(MAKE) $^
+
+temp_salt_ic.nc: gen_data ocean_hgrid.nc
+	$(LAUNCHER) ./gen_data
+sponge.nc: temp_salt_ic.nc
+	@test -f $@ || rm -f $^
+	@test -f $@ || $(MAKE) $^
+
+
+# Programs
 
 gen_grid: gen_grid.F90
 	$(FC) $(FCFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
@@ -20,6 +38,9 @@ gen_grid: gen_grid.F90
 gen_data: gen_data.F90
 	$(FC) $(FCFLAGS) $(LDFLAGS) -o $@ $^ $(LIBS)
 
+
+# Support
+
 .PHONY: clean
 clean:
 	rm -rf $(OUT) gen_grid gen_data