RFC: better debugging

configure.ac

@@ -889,6 +889,30 @@ AC_ARG_ENABLE([extra-warning],
 	[],
 	[enable_extra_warning=no])
 
+# Check for backtrace (requires glibc):
+AC_ARG_ENABLE([backtrace],
+	[AS_HELP_STRING([--enable-backtrace],
+		[print a stack trace on Terminate @<:@default=no@:>@])],
+	[AS_IF(
+		[test "x$enableval" = xyes], [enable_backtrace=yes],
+		[test "x$enableval" = xno], [enable_backtrace=no],
+		[enable_backtrace=no]
+	)],
+	[enable_backtrace=no])
+AS_IF([test "x$enable_backtrace" != xno],
+	[flag=:
+	AS_IF([$flag], [AC_SEARCH_LIBS([backtrace], [c], [], [flag=false])])
+	AS_IF([$flag], [AC_CHECK_HEADERS([execinfo.h], [], [flag=false])])
+	AS_IF([$flag],
+		[AC_DEFINE(ENABLE_BACKTRACE, [], [Define to print a stack trace on Terminate.])
+			enable_backtrace=yes],
+		[AS_IF([test "x$enable_backtrace" = xyes],
+			[AC_MSG_FAILURE([test for backtrace failed. Give --disable-backtrace if you want to compile without backtrace.])])
+		AC_MSG_NOTICE([backtrace is not available])
+		enable_backtrace=no])
+	]
+)
+
 # Optimization/debugging flags
 AC_ARG_VAR([COMPILEFLAGS], [Compiler flags for release versions])
 AC_ARG_VAR([LINKFLAGS], [Linker flags for release versions])
@@ -913,9 +937,9 @@ if test "$my_test_COMPILEFLAGS" != set; then
 		AX_HANDLE_EXTRA_WARNING([COMPILEFLAGS])
 		# Enable optimizations.
 		COMPILEFLAGS="$COMPILEFLAGS -O3"
-		if test "x$enable_profile" != xgprof; then
-			# -pg conflicts with -fomit-frame-pointer.
-			COMPILEFLAGS="$COMPILEFLAGS -fomit-frame-pointer"
+		if test "x$enable_backtrace" = xyes; then
+			# Keep symbols and frame pointers for easier debugging.
+			COMPILEFLAGS="$COMPILEFLAGS -g -fno-omit-frame-pointer"
 		fi
 		if test "x$enable_native" = xyes; then
 			# Use -march=native if available.
@@ -957,16 +981,21 @@ if test "$my_test_COMPILEFLAGS" != set; then
 fi
 my_test_LINKFLAGS=${LINKFLAGS+set}
 if test "$my_test_LINKFLAGS" != set; then
+	LINKFLAGS=""
+	if test "x$enable_backtrace" = xyes; then
+		# For easier debugging
+		LINKFLAGS="$LINKFLAGS -rdynamic"
+	fi
 	if test "x$vendor" = xgnu && test "x$print_os" = xOSX; then
 		# On OS X Mavericks, -s option has a funny effect: though the linker
 		# warns the option is obsolete and being ignored, it causes an internal
 		# error "atom not found in symbolIndex...".
-		LINKFLAGS=
+		LINKFLAGS="$LINKFLAGS"
 	elif test "x$enable_profile" != xno && test "x$enable_profile" != xunavailable; then
 		# Profilers needs symbol tables.
-		LINKFLAGS=
+		LINKFLAGS="$LINKFLAGS"
 	else
-		LINKFLAGS=-s
+		LINKFLAGS="$LINKFLAGS"
 	fi
 fi
 my_test_DEBUGCOMPILEFLAGS=${DEBUGCOMPILEFLAGS+set}
@@ -989,7 +1018,7 @@ if test "$my_test_DEBUGCOMPILEFLAGS" != set && test "x$enable_debug" = xyes; the
 			[DEBUGCOMPILEFLAGS="$DEBUGCOMPILEFLAGS -O0"],
 			[-Werror])
 		# Debugging information.
-		DEBUGCOMPILEFLAGS="$DEBUGCOMPILEFLAGS -g3"
+		DEBUGCOMPILEFLAGS="$DEBUGCOMPILEFLAGS -g3 -fno-omit-frame-pointer"
 		# Coverage option.
 		if test "x$enable_coverage" = xyes; then
 			DEBUGCOMPILEFLAGS="$DEBUGCOMPILEFLAGS -coverage"
@@ -1037,7 +1066,7 @@ if test "$my_test_DEBUGCOMPILEFLAGS" != set && test "x$enable_debug" = xyes; the
 fi
 my_test_DEBUGLINKFLAGS=${DEBUGLINKFLAGS+set}
 if test "$my_test_DEBUGLINKFLAGS" != set && test "x$enable_debug" = xyes; then
-	DEBUGLINKFLAGS=
+	DEBUGLINKFLAGS=-rdynamic
 	if test "x$vendor" = xgnu; then
 		# Coverage option.
 		if test "x$enable_coverage" = xyes; then
@@ -1156,6 +1185,16 @@ if test $atleastone = no; then
 	echo "    <NONE>"
 fi
 echo
+echo "Optionally enabled features:"
+atleastone=no
+if test "x$enable_backtrace" = xyes; then
+	echo "    backtrace"
+	atleastone=yes
+fi
+if test $atleastone = no; then
+	echo "    <NONE>"
+fi
+echo
 echo "The following executables can be compiled:"
 atleastone=no
 if test "x$build_form" = xyes; then

doc/manual/statements.tex

@@ -3758,6 +3758,10 @@ \section{off}
 \leftvitem{3.5cm}{allwarnings\index{off!allwarnings}}
 \rightvitem{13cm}{Turns off the printing of all warnings.}
 
+\leftvitem{3.5cm}{backtrace\index{off!backtrace}}
+\rightvitem{13cm}{Disables the printing of a stack trace on termination. This
+is the default for normal form builds.}
+
 \leftvitem{3.5cm}{checkpoint\index{off!checkpoint}}
 \rightvitem{13cm}{Deactivates the checkpoint mechanism. See
 \ref{checkpoints}.}
@@ -3894,6 +3898,12 @@ \section{on}
 \rightvitem{13cm}{Puts the printing of warnings in a mode in which all 
 warnings, even the very unimportant warnings are printed.}
 
+\leftvitem{3.5cm}{backtrace\index{on!backtrace}}
+\rightvitem{13cm}{Attempt to print a stack trace on termination, to assist with
+debugging. This is off by default for normal form builds and on by default for
+the debug binaries (vorm, tvorm, parvorm). For best results eu-addr2line or
+addr2line should be installed on the system.}
+
 \leftvitem{3.5cm}{checkpoint\index{on!checkpoint}}
 \rightvitem{13cm}{Activates the checkpoint mechanism that allows for
 the recovery of a crashed \FORM\ session. See \ref{checkpoints} for

sources/compcomm.c

@@ -137,6 +137,7 @@ static KEYWORDV onoffoptions[] = {
 	,{"innertest",      &(AC.InnerTest),  1,  0}
 	,{"wtimestats",     &(AC.WTimeStatsFlag),  1,  0}
 	,{"sortreallocate",	&(AC.SortReallocateFlag), 1, 0}
+	,{"backtrace",		&(AC.PrintBacktraceFlag), 1, 0}
 };
 
 static WORD one = 1;
@@ -671,7 +672,12 @@ int CoOn(UBYTE *s)
 			MesPrint("&Unrecognized option in ON statement: %s",t);
 			*s = c; return(-1);
 		}
-		if ( StrICont(t,(UBYTE *)"compress") == 0 ) {
+		if ( StrICont(t,(UBYTE *)"backtrace") == 0 ) {
+#ifndef ENABLE_BACKTRACE
+			Warning("backtrace not supported on this platform");
+#endif
+		}
+		else if ( StrICont(t,(UBYTE *)"compress") == 0 ) {
 			AR.gzipCompress = 0;
 			*s = c;
 			while ( *s == ' ' || *s == ',' || *s == '\t' ) s++;
@@ -899,7 +905,7 @@ int CoOn(UBYTE *s)
 			}
 		}
 		else { *s = c; }
-	 	*onoffoptions[i].var = onoffoptions[i].type; 
+		*onoffoptions[i].var = onoffoptions[i].type;
 		AR.SortType = AC.SortType;
 		AC.mparallelflag = AC.parallelflag | AM.hparallelflag;
 	}

sources/declare.h

@@ -186,6 +186,7 @@
 
 #define PREV(x) prevorder?prevorder:x
 
+#define Terminate(x) do { TerminateImpl(x, __FILE__, __LINE__, __FUNCTION__); } while(0)
 #define SETERROR(x) { Terminate(-1); return(-1); }
 
 /* use this macro to avoid the unused parameter warning */
@@ -409,7 +410,7 @@ static inline ULONG LongAbs(LONG x)
  */
 static inline int UnsignedToInt(unsigned int x)
 {
-	extern void Terminate(int);
+	extern void TerminateImpl(int, const char*, int, const char*);
 	if ( x <= INT_MAX ) return(x);
 	if ( x >= (unsigned int)INT_MIN )
 		return((int)(x - (unsigned int)INT_MIN) + INT_MIN);
@@ -419,7 +420,7 @@ static inline int UnsignedToInt(unsigned int x)
 
 static inline WORD UWordToWord(UWORD x)
 {
-	extern void Terminate(int);
+	extern void TerminateImpl(int, const char*, int, const char*);
 	if ( x <= WORD_MAX_VALUE ) return(x);
 	if ( x >= (UWORD)WORD_MIN_VALUE )
 		return((WORD)(x - (UWORD)WORD_MIN_VALUE) + WORD_MIN_VALUE);
@@ -429,7 +430,7 @@ static inline WORD UWordToWord(UWORD x)
 
 static inline LONG ULongToLong(ULONG x)
 {
-	extern void Terminate(int);
+	extern void TerminateImpl(int, const char*, int, const char*);
 	if ( x <= LONG_MAX_VALUE ) return(x);
 	if ( x >= (ULONG)LONG_MIN_VALUE )
 		return((LONG)(x - (ULONG)LONG_MIN_VALUE) + LONG_MIN_VALUE);
@@ -790,7 +791,7 @@ extern VOID   PositionStream(STREAM *,LONG);
 extern int    ReverseStatements(STREAM *);
 extern int    ProcessOption(UBYTE *,UBYTE *,int);
 extern int    DoSetups(VOID);
-extern VOID   Terminate(int);
+extern VOID   TerminateImpl(int, const char *,int, const char *);
 extern NAMENODE *GetNode(NAMETREE *,UBYTE *);
 extern int    AddName(NAMETREE *,UBYTE *,WORD,WORD,int *);
 extern int    GetName(NAMETREE *,UBYTE *,WORD *,int);

sources/startup.c

@@ -43,6 +43,13 @@
 #else
 #include <signal.h>
 #endif
+#ifdef ENABLE_BACKTRACE
+	#include <execinfo.h>
+#ifdef LINUX
+	#include <stdint.h>
+	#include <inttypes.h>
+#endif
+#endif
 
 /*
  * A macro for translating the contents of `x' into a string after expanding.
@@ -1386,6 +1393,13 @@ WORD IniVars(VOID)
 	AC.lUnitTrace = AM.gUnitTrace = 4;
 	AC.NamesFlag = AM.gNamesFlag = 0;
 	AC.CodesFlag = AM.gCodesFlag = 0;
+	/* Printing a backtrace on crash is on by default for both normal and debug
+		modes if FORM has been compiled with backtrace support. */
+#ifdef ENABLE_BACKTRACE
+	AC.PrintBacktraceFlag = 1;
+#else
+	AC.PrintBacktraceFlag = 0;
+#endif
 	AC.extrasymbols = AM.gextrasymbols = AM.ggextrasymbols = 0;
 	AC.extrasym = (UBYTE *)Malloc1(2*sizeof(UBYTE),"extrasym");
 	AM.gextrasym = (UBYTE *)Malloc1(2*sizeof(UBYTE),"extrasym");
@@ -1587,7 +1601,7 @@ static VOID onErrSig(int i)
 #endif
 	}
 	trappedTerminate = 1;
-	/*[13jul2005 mt]*//*Terminate(-1) on signal is here:*/
+	/*[13jul2005 mt]*//*TerminateImpl(-1) on signal is here:*/
 	Terminate(-1);
 }
 
@@ -1821,22 +1835,139 @@ dontremove:;
 
 /*
  		#] CleanUp : 
- 		#[ Terminate :
+ 		#[ TerminateImpl :
 */
 
 static int firstterminate = 1;
 
-VOID Terminate(int errorcode)
+VOID TerminateImpl(int errorcode, const char* file, int line, const char* function)
 {
 	if ( errorcode && firstterminate ) {
 		firstterminate = 0;
+
+		MLOCK(ErrorMessageLock);
 #ifdef WITHPTHREADS
 		MesPrint("Program terminating in thread %w at &");
 #elif defined(WITHMPI)
 		MesPrint("Program terminating in process %w at &");
 #else
 		MesPrint("Program terminating at &");
 #endif
+		MesPrint("Terminate called from %s:%d (%s)", file, line, function);
+
+		if ( AC.PrintBacktraceFlag ) {
+#ifdef ENABLE_BACKTRACE
+			void *stack[64];
+			int stacksize, stop = 0;
+			stacksize = backtrace(stack, sizeof(stack)/sizeof(stack[0]));
+
+			/* First check whether eu-addr2line is available */
+			if ( !system("command -v eu-addr2line > /dev/null 2>&1") ) {
+				MesPrint("Backtrace:");
+				for (int i = 0; i < stacksize && !stop; i++) {
+					FILE *fp;
+					char cmd[512];
+					// Leave an initial space
+					cmd[0] = ' ';
+					MesPrint("%#%2d:%", i);
+					snprintf(cmd+1, sizeof(cmd)-1, "eu-addr2line -s --pretty-print -f -i '%p' --pid=%d\n", stack[i], getpid());
+					fp = popen(cmd+1, "r");
+					while ( fgets(cmd+1, sizeof(cmd)-1, fp) != NULL ) {
+						MesPrint("%s", cmd);
+						/* Don't show functions lower than "main" (or thread equivalent) */
+						if ( strstr(cmd, " main ") || strstr(cmd, " RunThread ") || strstr(cmd, " RunSortBot ") ) {
+							stop = 1;
+						}
+					}
+					pclose(fp);
+				}
+			}
+#ifdef LINUX
+			else if ( !system("command -v addr2line > /dev/null 2>&1") ) {
+				/* Get the executable path. */
+				char exe_path[PATH_MAX];
+				{
+					ssize_t len = readlink("/proc/self/exe", exe_path, sizeof(exe_path) - 1);
+					if ( len != -1 ) {
+						exe_path[len] = '\0';
+					}
+					else {
+						goto backtrace_fallback;
+					}
+				}
+				/* Assume PIE binary and get the base address. */
+				uintptr_t base_address = 0;
+				{
+					char line[256];
+					FILE *maps = fopen("/proc/self/maps", "r");
+					if ( !maps ) {
+						goto backtrace_fallback;
+					}
+					/* See the format used by nommu_region_show() in fs/proc/nommu.c of the Linux source. */
+					if ( fgets(line, sizeof(line), maps) ) {
+						sscanf(line, "%" SCNxPTR "-", &base_address);
+					}
+					else {
+						fclose(maps);
+						goto backtrace_fallback;
+					}
+					fclose(maps);
+				}
+				char **strings;
+				strings = backtrace_symbols(stack, stacksize);
+				MesPrint("Backtrace:");
+				for ( int i = 0; i < stacksize && !stop; i++ ) {
+					FILE *fp;
+					char cmd[PATH_MAX + 512];
+					// Leave an initial space
+					cmd[0] = ' ';
+					uintptr_t addr = (uintptr_t)stack[i] - base_address;
+					MesPrint("%#%2d:%", i);
+					snprintf(cmd+1, sizeof(cmd)-1, "addr2line -e \"%s\" -i -p -s -f -C 0x%" PRIxPTR, exe_path, addr);
+					fp = popen(cmd+1, "r");
+					while ( fgets(cmd+1, sizeof(cmd)-1, fp) != NULL ) {
+						MesPrint("%s", cmd);
+						/* Don't show functions lower than "main" */
+						if ( strstr(cmd, " main ") || strstr(cmd, " RunThread ") || strstr(cmd, " RunSortBot ") ) {
+							stop = 1;
+						}
+					}
+					pclose(fp);
+				}
+				free(strings);
+			}
+#endif
+			else {
+				/* eu-addr2line not found */
+#ifdef LINUX
+backtrace_fallback: ;
+#endif
+				char **strings;
+				strings = backtrace_symbols(stack, stacksize);
+				MesPrint("Backtrace:");
+				for ( int i = 0; i < stacksize && !stop; i++ ) {
+					char *p = strings[i];
+					while ( *p && *p != '(' ) p++;
+					MesPrint("%#%2d: %s\n", i, p);
+					/* Don't show functions lower than "main" (or thread equivalent) */
+					if ( strstr(p, "(main+") || strstr(p, "(RunThread+") || strstr(p, "(RunSortBot+") ) {
+						stop = 1;
+					}
+				}
+#ifdef LINUX
+				MesPrint("Please install addr2line or eu-addr2line for readable stack information.");
+#else
+				MesPrint("Please install eu-addr2line for readable stack information.");
+#endif
+				free(strings);
+			}
+#else
+			MesPrint("FORM compiled without backtrace support.");
+#endif
+		} /* if ( AC.PrintBacktraceFlag) { */
+
+		MUNLOCK(ErrorMessageLock);
+
 		Crash();
 	}
 #ifdef TRAPSIGNALS
@@ -1905,7 +2036,7 @@ VOID Terminate(int errorcode)
 }
 
 /*
- 		#] Terminate : 
+ 		#] TerminateImpl : 
  		#[ PrintDeprecation :
 */