Split the large sort allocation into separate allocations.

Buffer overruns in these allocations will be visible to valgrind.

Fix units mismatch re: IObuffersize, IOtry.

Print warnings in debug mode, if any buffer sizes are altered by AllocSort.

Update "default" buffer sizes, so that we have no warnings.

Author: jodavies

sources/form3.h

@@ -438,6 +438,8 @@ template<typename T> struct calc {
 */
 #define WITHSORTBOTS
 
+#define SPLITALLOC
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

sources/fsizes.h

@@ -126,10 +126,10 @@
 #define MAXFPATCHES 256
 #define SORTIOSIZE 200000L
 
-#define SSMALLBUFFER 500000L
-#define SSMALLOVERFLOW 800000L
+#define SSMALLBUFFER 2560016L
+#define SSMALLOVERFLOW 3840032L
 #define STERMSSMALL 10000L
-#define SLARGEBUFFER 4000000L
+#define SLARGEBUFFER 26880512L
 #define SMAXPATCHES 64
 #define SMAXFPATCHES 64
 #define SSORTIOSIZE 32768L

sources/setfile.c

@@ -574,6 +574,7 @@ int AllocSetups(VOID)
 	AM.S0 = 0;
 	AM.S0 = AllocSort(LargeSize,SmallSize,SmallEsize,TermsInSmall
 					,MaxPatches,MaxFpatches,IOsize);
+	/* AM.S0->file.ziosize was already set to a (larger) value by AllocSort, here it is re-set. */
 #ifdef WITHZLIB
 	AM.S0->file.ziosize = IOsize;
 #ifndef WITHPTHREADS
@@ -848,20 +849,30 @@ VOID WriteSetup(VOID)
 		To be used for the main allocation of the sort buffers, and
 		in a later stage for the function and subroutine sort buffers.
 */
-
-SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsInSmall,
-                   int MaxPatches, int MaxFpatches, LONG IOsize)
+SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG inTermsInSmall,
+                   int inMaxPatches, int inMaxFpatches, LONG inIOsize)
 {
-	LONG allocation,longer,terms2insmall,sortsize,longerp;
+	LONG LargeSize = inLargeSize;
+	LONG SmallSize = inSmallSize;
+	LONG SmallEsize = inSmallEsize;
+	LONG TermsInSmall = inTermsInSmall;
+	int MaxPatches = inMaxPatches;
+	int MaxFpatches = inMaxFpatches;
+	LONG IOsize = inIOsize;
+
+#ifndef SPLITALLOC
+	LONG allocation;
+#endif
+	LONG longer,terms2insmall,sortsize,longerp;
 	LONG IObuffersize = IOsize;
 	LONG IOtry;
 	SORTING *sort;
-	int i = 0, j = 0;
+	int fname2Size = 0, j = 0;
 	char *s;
 	if ( AM.S0 != 0 ) {
-		s = FG.fname2; i = 0;
-		while ( *s ) { s++; i++; }
-		i += 16;
+		s = FG.fname2; fname2Size = 0;
+		while ( *s ) { s++; fname2Size++; }
+		fname2Size += 16;
 	}
 	if ( MaxFpatches < 4 ) MaxFpatches = 4;
 	longer = MaxPatches > MaxFpatches ? MaxPatches : MaxFpatches;
@@ -876,7 +887,6 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 	terms2insmall = 2*TermsInSmall;  /* Used to be just + 100 rather than *2 */
 	if ( SmallEsize < (SmallSize*3)/2 ) SmallEsize = (SmallSize*3)/2;
 	if ( LargeSize > 0 && LargeSize < 2*SmallSize ) LargeSize = 2*SmallSize;
-/*	if ( SmallEsize < 3*AM.MaxTer ) SmallEsize = 3*AM.MaxTer; */
 	SmallEsize = (SmallEsize+15) & (-16L);
 	if ( LargeSize < 0 ) LargeSize = 0;
 	sortsize = sizeof(SORTING);
@@ -898,9 +908,23 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 
 	IOtry = ((LargeSize+SmallEsize)/MaxFpatches-2*AM.MaxTer)/sizeof(WORD)-COMPINC;
 
-	if ( (LONG)(IObuffersize*sizeof(WORD)) < IOtry )
-		IObuffersize = (IOtry+sizeof(WORD)-1)/sizeof(WORD);
+	/* Here both IObuffersize and IOtry are in units of sizeof(WORD). */
+	if ( IObuffersize < IOtry ) {
+		IObuffersize = IOtry;
+	}
+
+#if DEBUGGING
+	if ( LargeSize != inLargeSize ) { MesPrint("Warning: LargeSize adjusted: %l -> %l", inLargeSize, LargeSize); }
+	if ( SmallSize != inSmallSize ) { MesPrint("Warning: SmallSize adjusted: %l -> %l", inSmallSize, SmallSize); }
+	if ( SmallEsize != inSmallEsize ) {MesPrint("Warning: SmallEsize adjusted: %l -> %l", inSmallEsize, SmallEsize); }
+	if ( TermsInSmall != inTermsInSmall ) { MesPrint("Warning: TermsInSmall adjusted: %l -> %l", inTermsInSmall, TermsInSmall); }
+	if ( MaxPatches != inMaxPatches ) { MesPrint("Warning: MaxPatches adjusted: %d -> %d", inMaxPatches, MaxPatches); }
+	if ( MaxFpatches != inMaxFpatches ) {MesPrint("Warning: MaxFPatches adjusted: %d -> %d", inMaxFpatches, MaxFpatches); }
+	/* This one is always changed if the LargeSize has not been... */
+	/* if ( IObuffersize != inIOsize/(LONG)sizeof(WORD) ) { MesPrint("Warning: IOsize adjusted: %l -> %l", inIOsize/sizeof(WORD), IObuffersize); } */
+#endif
 
+#ifndef SPLITALLOC
 	allocation =
 		 3*sizeof(POSITION)*(LONG)longer				/* Filepositions!! */
 		+2*sizeof(WORD *)*longer
@@ -914,7 +938,7 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 		+LargeSize
 		+SmallEsize
 		+sortsize
-		+IObuffersize*sizeof(WORD) + i + 16;
+		+IObuffersize*sizeof(WORD) + fname2Size + 16;
 	sort = (SORTING *)Malloc1(allocation,"sort buffers");
 
 	sort->LargeSize = LargeSize/sizeof(WORD);
@@ -931,11 +955,13 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 	sort->pStop = sort->Patches+longer;
 	sort->poina = sort->pStop+longer;
 	sort->poin2a = sort->poina + longerp;
+
 	sort->fPatches = (POSITION *)(sort->poin2a+longerp);
 	sort->fPatchesStop = sort->fPatches + longer;
 	sort->inPatches = sort->fPatchesStop + longer;
 	sort->tree = (WORD *)(sort->inPatches + longer);
 	sort->used = sort->tree+longerp;
+
 #ifdef WITHZLIB
 	sort->fpcompressed = sort->used+longerp;
 	sort->fpincompressed = sort->fpcompressed+longerp+2;
@@ -944,13 +970,16 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 #else
 	sort->ktoi = sort->used + longerp;
 #endif
+
 	sort->lBuffer = (WORD *)(sort->ktoi + longerp + 2);
 	sort->lTop = sort->lBuffer+sort->LargeSize;
+
 	sort->sBuffer = sort->lTop;
 	if ( sort->LargeSize == 0 ) { sort->lBuffer = 0; sort->lTop = 0; }
 	sort->sTop = sort->sBuffer + sort->SmallSize;
 	sort->sTop2 = sort->sBuffer + sort->SmallEsize;
 	sort->sHalf = sort->sBuffer + (LONG)((sort->SmallSize+sort->SmallEsize)>>1);
+
 	sort->file.PObuffer = (WORD *)(sort->sTop2);
 	sort->file.POstop = sort->file.PObuffer+IObuffersize;
 	sort->file.POsize = IObuffersize * sizeof(WORD);
@@ -975,6 +1004,77 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 	sort->cBufferSize = 0;
 	sort->f = 0;
 	sort->PolyWise = 0;
+#endif
+
+
+#ifdef SPLITALLOC
+	// Separate buffers.
+	sort = Malloc1(sizeof(*sort), "SPLITALLOC sorting struct");
+
+	sort->LargeSize = LargeSize/sizeof(WORD);
+	sort->SmallSize = SmallSize/sizeof(WORD);
+	sort->SmallEsize = SmallEsize/sizeof(WORD);
+	sort->MaxPatches = MaxPatches;
+	sort->MaxFpatches = MaxFpatches;
+	sort->TermsInSmall = TermsInSmall;
+	sort->Terms2InSmall = terms2insmall;
+
+	sort->sPointer     = Malloc1(sizeof(*(sort->sPointer    ))*terms2insmall, "SPLITALLOC sPointer");
+	sort->SplitScratch = Malloc1(sizeof(*(sort->SplitScratch))*terms2insmall/2, "SPLITALLOC SplitScratch");
+	sort->Patches      = Malloc1(sizeof(*(sort->Patches     ))*longer, "SPLITALLOC Patches");
+	sort->pStop        = Malloc1(sizeof(*(sort->pStop       ))*longer, "SPLITALLOC pStop");
+	sort->poina        = Malloc1(sizeof(*(sort->poina       ))*longerp, "SPLITALLOC poina");
+	sort->poin2a       = Malloc1(sizeof(*(sort->poin2a      ))*longerp, "SPLITALLOC poin2a");
+
+	sort->fPatches     = Malloc1(sizeof(*(sort->fPatches    ))*longer, "SPLITALLOC fPatches");
+	sort->fPatchesStop = Malloc1(sizeof(*(sort->fPatchesStop))*longer, "SPLITALLOC fPatchesStop");
+	sort->inPatches    = Malloc1(sizeof(*(sort->inPatches   ))*longer, "SPLITALLOC inPatches");
+	sort->tree         = Malloc1(sizeof(*(sort->tree        ))*longerp, "SPLITALLOC tree");
+	sort->used         = Malloc1(sizeof(*(sort->used        ))*longerp, "SPLITALLOC used");
+
+#ifdef WITHZLIB
+	sort->fpcompressed   = Malloc1(sizeof(*(sort->fpcompressed  ))*(longerp+2), "SPLITALLOC fpcompressed");
+	sort->fpincompressed = Malloc1(sizeof(*(sort->fpincompressed))*(longerp+2), "SPLITALLOC fpincompressed");
+	sort->zsparray = 0;
+#endif
+
+	sort->ktoi         = Malloc1(sizeof(*(sort->ktoi))*(longerp+2), "SPLITALLOC ktoi");
+
+	// The combined Large buffer and Small buffer (+ extension) are used.
+	// They must be allocated together.
+	sort->lBuffer      = Malloc1(sizeof(*(sort->lBuffer))*(sort->LargeSize+sort->SmallEsize), "SPLITALLOC lBuffer+sBuffer");
+	sort->lTop = sort->lBuffer+sort->LargeSize;
+
+	sort->sBuffer = sort->lTop;
+	if ( sort->LargeSize == 0 ) { sort->lBuffer = 0; sort->lTop = 0; }
+	sort->sTop = sort->sBuffer + sort->SmallSize;
+	sort->sTop2 = sort->sBuffer + sort->SmallEsize;
+	sort->sHalf = sort->sBuffer + (LONG)((sort->SmallSize+sort->SmallEsize)>>1);
+
+	sort->file.PObuffer = Malloc1(IObuffersize*sizeof(*(sort->file.PObuffer))+fname2Size+16, "SPLITALLOC PObuffer");
+	sort->file.POstop = sort->file.PObuffer+IObuffersize;
+	sort->file.POsize = IObuffersize * sizeof(WORD);
+	sort->file.POfill = sort->file.POfull = sort->file.PObuffer;
+	sort->file.active = 0;
+	sort->file.handle = -1;
+	PUTZERO(sort->file.POposition);
+#ifdef WITHPTHREADS
+	sort->file.pthreadslock = dummylock;
+#endif
+#ifdef WITHZLIB
+	sort->file.ziosize = IObuffersize*sizeof(WORD);
+	sort->file.ziobuffer = 0;
+#endif
+	if ( AM.S0 != 0 ) {
+		sort->file.name = (char *)(sort->file.PObuffer + IObuffersize);
+		AllocSortFileName(sort);
+	}
+	else sort->file.name = 0;
+	sort->cBuffer = 0;
+	sort->cBufferSize = 0;
+	sort->f = 0;
+	sort->PolyWise = 0;
+#endif
 
 	return(sort);
 }

sources/sort.c

@@ -4757,7 +4757,27 @@ void CleanUpSort(int num)
 					MUNLOCK(ErrorMessageLock);
 #endif
 				}
+#ifdef SPLITALLOC
+				M_free(S->sPointer, "SPLITALLOC sPointer");
+				M_free(S->SplitScratch, "SPLITALLOC SplitScratch");
+				M_free(S->Patches, "SPLITALLOC Patches");
+				M_free(S->pStop, "SPLITALLOC pStop");
+				M_free(S->poina, "SPLITALLOC poina");
+				M_free(S->poin2a, "SPLITALLOC poin2a");
+				M_free(S->fPatches, "SPLITALLOC fPatches");
+				M_free(S->fPatchesStop, "SPLITALLOC fPatchesStop");
+				M_free(S->inPatches, "SPLITALLOC inPatches");
+				M_free(S->tree, "SPLITALLOC tree");
+				M_free(S->used, "SPLITALLOC used");
+				M_free(S->fpcompressed, "SPLITALLOC fpcompressed");
+				M_free(S->fpincompressed, "SPLITALLOC fpincompressed");
+				M_free(S->ktoi, "SPLITALLOC ktoi");
+				M_free(S->lBuffer, "SPLITALLOC lBuffer+sBuffer");
+				M_free(S->file.PObuffer, "SPLITALLOC PObuffer");
+				M_free(S, "SPLITALLOC sorting struct");
+#else
 				M_free(S,"sorting struct");
+#endif
 			}
 			AN.FunSorts[i] = 0;
 		}