From a20d928ef6e35a3c31accd5e87178a04b0ab3041 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer <eisen@cc.gatech.edu> Date: Sun, 26 Jun 2022 22:07:04 -0400 Subject: [PATCH 1/6] BP5 subformats --- source/adios2/toolkit/format/bp5/BP5Base.cpp | 101 ++++++++++++ source/adios2/toolkit/format/bp5/BP5Base.h | 57 +++++-- .../toolkit/format/bp5/BP5Deserializer.cpp | 83 ++++------ .../toolkit/format/bp5/BP5Deserializer.h | 5 +- .../toolkit/format/bp5/BP5Serializer.cpp | 156 +++++++++--------- 5 files changed, 254 insertions(+), 148 deletions(-) diff --git a/source/adios2/toolkit/format/bp5/BP5Base.cpp b/source/adios2/toolkit/format/bp5/BP5Base.cpp index b8ae2dfb52..b74a1498a0 100644 --- a/source/adios2/toolkit/format/bp5/BP5Base.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Base.cpp @@ -49,5 +49,106 @@ int BP5Base::BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit) const return ((MBase->BitField[Element] & ((size_t)1 << ElementBit)) == ((size_t)1 << ElementBit)); } +#define BASE_FIELD_ENTRIES \ + {"Dims", "integer", sizeof(size_t), \ + FMOffset(BP5Base::MetaArrayRec *, Dims)}, \ + {"BlockCount", "integer", sizeof(size_t), \ + FMOffset(BP5Base::MetaArrayRec *, BlockCount)}, \ + {"DBCount", "integer", sizeof(size_t), \ + FMOffset(BP5Base::MetaArrayRec *, DBCount)}, \ + {"Shape", "integer[Dims]", sizeof(size_t), \ + FMOffset(BP5Base::MetaArrayRec *, Shape)}, \ + {"Count", "integer[DBCount]", sizeof(size_t), \ + FMOffset(BP5Base::MetaArrayRec *, Count)}, \ + {"Offset", "integer[DBCount]", sizeof(size_t), \ + FMOffset(BP5Base::MetaArrayRec *, Offsets)}, \ + {"DataBlockLocation", "integer[BlockCount]", sizeof(size_t), \ + FMOffset(BP5Base::MetaArrayRec *, DataBlockLocation)}, + +static FMField MetaArrayRecList[] = {BASE_FIELD_ENTRIES{NULL, NULL, 0, 0}}; + +static FMField MetaArrayRecOperatorList[] = { + BASE_FIELD_ENTRIES{ + "DataBlockSize", "integer[BlockCount]", sizeof(size_t), + FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)}, + {NULL, NULL, 0, 0}}; + +static FMField MetaArrayRecMM1List[] = { + BASE_FIELD_ENTRIES{"MinMax", "char[2][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecMM *, MinMax)}, + {NULL, NULL, 0, 0}}; + +static FMField MetaArrayRecOperatorMM1List[] = { + BASE_FIELD_ENTRIES{ + "DataBlockSize", "integer[BlockCount]", sizeof(size_t), + FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)}, + {"MinMax", "char[2][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecMM *, MinMax)}, + {NULL, NULL, 0, 0}}; +static FMField MetaArrayRecMM2List[] = { + BASE_FIELD_ENTRIES{"MinMax", "char[4][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecMM *, MinMax)}, + {NULL, NULL, 0, 0}}; + +static FMField MetaArrayRecOperatorMM2List[] = { + BASE_FIELD_ENTRIES{ + "DataBlockSize", "integer[BlockCount]", sizeof(size_t), + FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)}, + {"MinMax", "char[4][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)}, + {NULL, NULL, 0, 0}}; +static FMField MetaArrayRecMM4List[] = { + BASE_FIELD_ENTRIES{"MinMax", "char[8][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecMM *, MinMax)}, + {NULL, NULL, 0, 0}}; + +static FMField MetaArrayRecOperatorMM4List[] = { + BASE_FIELD_ENTRIES{ + "DataBlockSize", "integer[BlockCount]", sizeof(size_t), + FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)}, + {"MinMax", "char[8][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)}, + {NULL, NULL, 0, 0}}; +static FMField MetaArrayRecMM8List[] = { + BASE_FIELD_ENTRIES{"MinMax", "char[16][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecMM *, MinMax)}, + {NULL, NULL, 0, 0}}; + +static FMField MetaArrayRecOperatorMM8List[] = { + BASE_FIELD_ENTRIES{ + "DataBlockSize", "integer[BlockCount]", sizeof(size_t), + FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)}, + {"MinMax", "char[16][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)}, + {NULL, NULL, 0, 0}}; +static FMField MetaArrayRecMM16List[] = { + BASE_FIELD_ENTRIES{"MinMax", "char[32][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecMM *, MinMax)}, + {NULL, NULL, 0, 0}}; + +static FMField MetaArrayRecOperatorMM16List[] = { + BASE_FIELD_ENTRIES{ + "DataBlockSize", "integer[BlockCount]", sizeof(size_t), + FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)}, + {"MinMax", "char[32][BlockCount]", 1, + FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)}, + {NULL, NULL, 0, 0}}; +#undef BASE_FIELD_ENTRIES + +BP5Base::BP5Base() +{ + MetaArrayRecListPtr = &MetaArrayRecList[0]; + MetaArrayRecOperatorListPtr = &MetaArrayRecOperatorList[0]; + MetaArrayRecMM1ListPtr = &MetaArrayRecMM1List[0]; + MetaArrayRecOperatorMM1ListPtr = &MetaArrayRecOperatorMM1List[0]; + MetaArrayRecMM2ListPtr = &MetaArrayRecMM2List[0]; + MetaArrayRecOperatorMM2ListPtr = &MetaArrayRecOperatorMM2List[0]; + MetaArrayRecMM4ListPtr = &MetaArrayRecMM4List[0]; + MetaArrayRecOperatorMM4ListPtr = &MetaArrayRecOperatorMM4List[0]; + MetaArrayRecMM8ListPtr = &MetaArrayRecMM8List[0]; + MetaArrayRecOperatorMM8ListPtr = &MetaArrayRecOperatorMM8List[0]; + MetaArrayRecMM16ListPtr = &MetaArrayRecMM16List[0]; + MetaArrayRecOperatorMM16ListPtr = &MetaArrayRecOperatorMM16List[0]; +} } } diff --git a/source/adios2/toolkit/format/bp5/BP5Base.h b/source/adios2/toolkit/format/bp5/BP5Base.h index 0d843a4813..d9b2d17785 100644 --- a/source/adios2/toolkit/format/bp5/BP5Base.h +++ b/source/adios2/toolkit/format/bp5/BP5Base.h @@ -27,6 +27,8 @@ namespace format class BP5Base { public: + BP5Base(); + struct MetaMetaInfoBlock { char *MetaMetaInfo; @@ -35,29 +37,42 @@ class BP5Base size_t MetaMetaIDLen; }; +#define BASE_FIELDS \ + size_t Dims; /* How many dimensions does this array have */ \ + size_t BlockCount; /* How many blocks are written */ \ + size_t DBCount; /* Dimens * BlockCount */ \ + size_t *Shape; /* Global dimensionality [Dims] NULL for local */ \ + size_t *Count; /* Per-block Counts [DBCount] */ \ + size_t *Offsets; /* Per-block Offsets [DBCount] NULL for local \ + */ \ + size_t *DataBlockLocation; /* Per-block Offset in PG [BlockCount] */ + typedef struct _MetaArrayRec { - size_t Dims; // How many dimensions does this array have - size_t BlockCount; // How many blocks are written - size_t DBCount; // Dimens * BlockCount - size_t *Shape; // Global dimensionality [Dims] NULL for local - size_t *Count; // Per-block Counts [DBCount] - size_t *Offsets; // Per-block Offsets [DBCount] NULL for local - size_t *DataLocation; // Per-block Offsets [BlockCount] + BASE_FIELDS } MetaArrayRec; typedef struct _MetaArrayRecOperator { - size_t Dims; // How many dimensions does this array have - size_t BlockCount; // How many blocks are written - size_t DBCount; // Dimens * BlockCount - size_t *Shape; // Global dimensionality [Dims] NULL for local - size_t *Count; // Per-block Counts [DBCount] - size_t *Offsets; // Per-block Offsets [DBCount] NULL for local - size_t *DataLocation; // Per-block Offsets [BlockCount] - size_t *DataLengths; // Per-block Lengths [BlockCount] + BASE_FIELDS + size_t *DataBlockSize; // Per-block Lengths [BlockCount] } MetaArrayRecOperator; + typedef struct _MetaArrayRecMM + { + BASE_FIELDS + char *MinMax; // char[TYPESIZE][BlockCount] varies by type + } MetaArrayRecMM; + + typedef struct _MetaArrayRecOperatorMM + { + BASE_FIELDS + size_t *DataBlockSize; // Per-block Lengths [BlockCount] + char *MinMax; // char[TYPESIZE][BlockCount] varies by type + } MetaArrayRecOperatorMM; + +#undef BASE_FIELDS + struct BP5MetadataInfoStruct { size_t BitFieldCount; @@ -67,6 +82,18 @@ class BP5Base void BP5BitfieldSet(struct BP5MetadataInfoStruct *MBase, int Bit) const; int BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit) const; + FMField *MetaArrayRecListPtr; + FMField *MetaArrayRecOperatorListPtr; + FMField *MetaArrayRecMM1ListPtr; + FMField *MetaArrayRecOperatorMM1ListPtr; + FMField *MetaArrayRecMM2ListPtr; + FMField *MetaArrayRecOperatorMM2ListPtr; + FMField *MetaArrayRecMM4ListPtr; + FMField *MetaArrayRecOperatorMM4ListPtr; + FMField *MetaArrayRecMM8ListPtr; + FMField *MetaArrayRecOperatorMM8ListPtr; + FMField *MetaArrayRecMM16ListPtr; + FMField *MetaArrayRecOperatorMM16ListPtr; }; } // end namespace format } // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 5a0f4fc7d9..9d10777857 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -171,39 +171,35 @@ void BP5Deserializer::BreakdownVarName(const char *Name, char **base_name_p, *base_name_p = strdup(NameStart); } +void BP5Deserializer::BreakdownFieldType(const char *FieldType, bool &Operator, + bool &MinMax) +{ + assert(FieldType[0] == 'M'); // sanity check, should start with "MetaArray" + FieldType += strlen("MetaArray"); + if (FieldType == 0) + return; + if (FieldType[0] == 'O') + { + Operator = true; + FieldType += strlen("Op"); + } + if (FieldType[0] == 'M') + { + MinMax = true; + } +} + void BP5Deserializer::BreakdownArrayName(const char *Name, char **base_name_p, - DataType *type_p, int *element_size_p, - char **Operator, bool *MinMax) + DataType *type_p, int *element_size_p) { int Type; int ElementSize; const char *NameStart = strchr(strchr(Name + 4, '_') + 1, '_') + 1; // + 3 to skip BP5_ or bp5_ prefix sscanf(Name + 4, "%d_%d", &ElementSize, &Type); - const char *Plus = index(Name, '+'); - *Operator = NULL; - *MinMax = false; - while (Plus && (*Plus == '+')) - { - int Len; - if (sscanf(Plus, "+%dO", &Len) == 1) - { // Operator Spec - *Operator = (char *)malloc(Len + 1); - const char *OpStart = index(Plus, 'O') + 1; - memcpy(*Operator, index(Plus, 'O') + 1, Len); - (*Operator)[Len] = 0; - Plus = OpStart + Len; - } - else if (strncmp(Plus, "+MM", 3) == 0) - { - *MinMax = true; - Plus += 3; - } - } *element_size_p = ElementSize; *type_p = (DataType)Type; *base_name_p = strdup(NameStart); - *(rindex(*base_name_p, '_')) = 0; } BP5Deserializer::BP5VarRec *BP5Deserializer::LookupVarByKey(void *Key) const @@ -283,10 +279,11 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) char *ArrayName; DataType Type; int ElementSize; - char *Operator = NULL; + bool Operator = false; bool MinMax = false; - BreakdownArrayName(FieldList[i + 4].field_name, &ArrayName, &Type, - &ElementSize, &Operator, &MinMax); + BreakdownFieldType(FieldList[i].field_type, Operator, MinMax); + BreakdownArrayName(FieldList[i].field_name, &ArrayName, &Type, + &ElementSize); VarRec = LookupVarByName(ArrayName); if (!VarRec) { @@ -294,7 +291,8 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) VarRec->Type = Type; VarRec->ElementSize = ElementSize; VarRec->OrigShapeID = C->OrigShapeID; - VarRec->Operator = Operator; + if (Operator) + VarRec->Operator = strdup("SomeOperator"); C->ElementSize = ElementSize; } C->VarRec = VarRec; @@ -307,9 +305,8 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) { VarRec->MinMaxOffset = MetaRecFields * sizeof(void *); - MetaRecFields++; } - i += MetaRecFields; + i++; free(ArrayName); } else @@ -658,24 +655,6 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, VarRec->PerWriterBlockStart[WriterRank + 1] = VarRec->PerWriterBlockStart[WriterRank] + BlockCount; } - if (VarRec->MinMaxOffset != SIZE_MAX) - { - MinMaxStruct MinMax; - MinMax.Init(VarRec->Type); - for (size_t B = 0; B < BlockCount; B++) - { - void *MMs = *(void **)(((char *)meta_base) + - VarRec->MinMaxOffset); - char *BlockMinAddr = - (((char *)MMs) + 2 * B * VarRec->ElementSize); - char *BlockMaxAddr = - (((char *)MMs) + (2 * B + 1) * VarRec->ElementSize); - ApplyElementMinMax(MinMax, VarRec->Type, - (void *)BlockMinAddr); - ApplyElementMinMax(MinMax, VarRec->Type, - (void *)BlockMaxAddr); - } - } } } else @@ -1121,7 +1100,7 @@ BP5Deserializer::GenerateReadRequests(const bool doAllocTempBuffers, RR.Timestep = Req->Step; RR.WriterRank = WriterRank; RR.StartOffset = - writer_meta_base->DataLocation[NeededBlock]; + writer_meta_base->DataBlockLocation[NeededBlock]; RR.ReadLength = helper::GetDataTypeSize(Req->VarRec->Type) * @@ -1179,9 +1158,9 @@ BP5Deserializer::GenerateReadRequests(const bool doAllocTempBuffers, RR.Timestep = Req->Step; RR.WriterRank = WriterRank; RR.StartOffset = - writer_meta_base->DataLocation[Block]; + writer_meta_base->DataBlockLocation[Block]; RR.ReadLength = - writer_meta_base->DataLengths[Block]; + writer_meta_base->DataBlockSize[Block]; RR.DestinationAddr = nullptr; if (doAllocTempBuffers) { @@ -1228,7 +1207,7 @@ BP5Deserializer::GenerateReadRequests(const bool doAllocTempBuffers, RR.Timestep = Req->Step; RR.WriterRank = WriterRank; RR.StartOffset = - writer_meta_base->DataLocation[Block] + + writer_meta_base->DataBlockLocation[Block] + StartOffsetInBlock; RR.ReadLength = EndOffsetInBlock - StartOffsetInBlock; @@ -1292,7 +1271,7 @@ void BP5Deserializer::FinalizeGet(const ReadRequest &Read, const bool freeAddr) std::lock_guard<std::mutex> lockGuard(mutexDecompress); core::Decompress(IncomingData, ((MetaArrayRecOperator *)writer_meta_base) - ->DataLengths[Read.BlockID], + ->DataBlockSize[Read.BlockID], decompressBuffer.data()); } IncomingData = decompressBuffer.data(); diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.h b/source/adios2/toolkit/format/bp5/BP5Deserializer.h index 528bdc13b6..0e4365a99c 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.h @@ -182,9 +182,10 @@ class BP5Deserializer : virtual public BP5Base void ReverseDimensions(size_t *Dimensions, int count, int times); void BreakdownVarName(const char *Name, char **base_name_p, DataType *type_p, int *element_size_p); + void BreakdownFieldType(const char *FieldType, bool &Operator, + bool &MinMax); void BreakdownArrayName(const char *Name, char **base_name_p, - DataType *type_p, int *element_size_p, - char **Operator, bool *MinMax); + DataType *type_p, int *element_size_p); void *VarSetup(core::Engine *engine, const char *variableName, const DataType type, void *data); void *ArrayVarSetup(core::Engine *engine, const char *variableName, diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index 5efbbc2226..f3d0455035 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -257,27 +257,12 @@ static char *BuildShortName(const ShapeID Shape, const int Index, } static char *BuildLongName(const char *base_name, const ShapeID Shape, - const int type, const int element_size, - const char *Operator, bool MinMax) + const int type, const int element_size) { const char *Prefix = NamePrefix(Shape); int Len = strlen(base_name) + 3 + strlen(Prefix) + 16; char *Ret = (char *)malloc(Len); sprintf(Ret, "%s_%d_%d", Prefix, element_size, type); - if (Operator && (strcmp(Operator, "") != 0)) - { - Len += - 5 + - strlen(Operator); // for "+%d%s" where %d is len of operator string - Ret = (char *)realloc(Ret, Len); - sprintf(&Ret[strlen(Ret)], "+%zuO%s", strlen(Operator), Operator); - } - if (MinMax) - { - Len += 3; - Ret = (char *)realloc(Ret, Len); - strcat(Ret, "+MM"); - } strcat(Ret, "_"); strcat(Ret, base_name); return Ret; @@ -453,63 +438,50 @@ BP5Serializer::CreateWriterRec(void *Variable, const char *Name, DataType Type, // Array field. To Metadata, add FMFields for DimCount, Shape, Count // and Offsets matching _MetaArrayRec char *LongName = - BuildLongName(Name, VB->m_ShapeID, (int)Type, ElemSize, - OperatorType, /* minmax */ (m_StatsLevel > 0)); - char *DimsName = BuildShortName(VB->m_ShapeID, Info.RecCount, "Dims"); - char *BlockCountName = - BuildShortName(VB->m_ShapeID, Info.RecCount, "BlockCount"); - char *ArrayDBCount = - BuildShortName(VB->m_ShapeID, Info.RecCount, "DBCount"); - char *CountName = ConcatName(LongName, "Count"); - char *ShapeName = BuildShortName(VB->m_ShapeID, Info.RecCount, "Shape"); - char *OffsetsName = - BuildShortName(VB->m_ShapeID, Info.RecCount, "Offsets"); - char *LocationsName = - BuildShortName(VB->m_ShapeID, Info.RecCount, "DataLocations"); - char *LengthsName = - BuildShortName(VB->m_ShapeID, Info.RecCount, "DataLengths"); - char *MinMaxName = - BuildShortName(VB->m_ShapeID, Info.RecCount, "MinMax"); - AddField(&Info.MetaFields, &Info.MetaFieldCount, DimsName, - DataType::Int64, sizeof(size_t)); - Rec->MetaOffset = Info.MetaFields[Info.MetaFieldCount - 1].field_offset; - AddField(&Info.MetaFields, &Info.MetaFieldCount, BlockCountName, - DataType::Int64, sizeof(size_t)); - AddField(&Info.MetaFields, &Info.MetaFieldCount, ArrayDBCount, - DataType::Int64, sizeof(size_t)); - AddFixedArrayField(&Info.MetaFields, &Info.MetaFieldCount, ShapeName, - DataType::Int64, sizeof(size_t), DimCount); - AddVarArrayField(&Info.MetaFields, &Info.MetaFieldCount, CountName, - DataType::Int64, sizeof(size_t), ArrayDBCount); - AddVarArrayField(&Info.MetaFields, &Info.MetaFieldCount, OffsetsName, - DataType::Int64, sizeof(size_t), ArrayDBCount); - AddVarArrayField(&Info.MetaFields, &Info.MetaFieldCount, LocationsName, - DataType::Int64, sizeof(size_t), BlockCountName); - size_t Offset = sizeof(MetaArrayRec); + BuildLongName(Name, VB->m_ShapeID, (int)Type, ElemSize); + + const char *ArrayTypeName = "MetaArray"; + int FieldSize = sizeof(MetaArrayRec); if (VB->m_Operations.size()) { - AddVarArrayField(&Info.MetaFields, &Info.MetaFieldCount, - LengthsName, DataType::Int64, sizeof(size_t), - BlockCountName); - Offset += sizeof(void *); + ArrayTypeName = "MetaArrayOp"; + FieldSize = sizeof(MetaArrayRecOperator); } if (m_StatsLevel > 0) { - Rec->MinMaxOffset = Offset; - AddDoubleArrayField(&Info.MetaFields, &Info.MetaFieldCount, - MinMaxName, Type, ElemSize, BlockCountName); + char MMArrayName[40] = {0}; + strcat(MMArrayName, ArrayTypeName); + switch (ElemSize) + { + case 1: + strcat(MMArrayName, "MM1"); + break; + case 2: + strcat(MMArrayName, "MM2"); + break; + case 4: + strcat(MMArrayName, "MM4"); + break; + case 8: + strcat(MMArrayName, "MM8"); + break; + case 16: + strcat(MMArrayName, "MM16"); + break; + } + Rec->MinMaxOffset = FieldSize; + FieldSize += sizeof(char *); + AddSimpleField(&Info.MetaFields, &Info.MetaFieldCount, LongName, + MMArrayName, FieldSize); + } + else + { + AddSimpleField(&Info.MetaFields, &Info.MetaFieldCount, LongName, + ArrayTypeName, FieldSize); } + Rec->MetaOffset = Info.MetaFields[Info.MetaFieldCount - 1].field_offset; Rec->OperatorType = OperatorType; free(LongName); - free(DimsName); - free(ArrayDBCount); - free(BlockCountName); - free(ShapeName); - free(CountName); - free(OffsetsName); - free(LocationsName); - free(LengthsName); - free(MinMaxName); RecalcMarshalStorageSize(); // Changing the formats renders these invalid @@ -562,7 +534,7 @@ void BP5Serializer::DumpDeferredBlocks(bool forceCopyDeferred) m_PriorDataBufferSizeTotal + CurDataBuffer->AddToVec(Def.DataSize, Def.Data, Def.AlignReq, forceCopyDeferred); - MetaEntry->DataLocation[Def.BlockID] = DataOffset; + MetaEntry->DataBlockLocation[Def.BlockID] = DataOffset; } DeferredExterns.clear(); } @@ -629,7 +601,7 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, /* * If this is a big external block, we'll do everything except add it to * the BufferV now, saving enough information to add it and patch back - * the DataLocation in the metadata in DumpDeferredBlocks() + * the DataBlockLocation in the metadata in DumpDeferredBlocks() */ DeferAddToVec = true; } @@ -730,14 +702,14 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, MetaEntry->DBCount = DimCount; MetaEntry->Count = CopyDims(DimCount, Count); MetaEntry->BlockCount = 1; - MetaEntry->DataLocation = (size_t *)malloc(sizeof(size_t)); - MetaEntry->DataLocation[0] = DataOffset; + MetaEntry->DataBlockLocation = (size_t *)malloc(sizeof(size_t)); + MetaEntry->DataBlockLocation[0] = DataOffset; if (Rec->OperatorType) { MetaArrayRecOperator *OpEntry = (MetaArrayRecOperator *)MetaEntry; - OpEntry->DataLengths = (size_t *)malloc(sizeof(size_t)); - OpEntry->DataLengths[0] = CompressedSize; + OpEntry->DataBlockSize = (size_t *)malloc(sizeof(size_t)); + OpEntry->DataBlockSize[0] = CompressedSize; } if (Offsets) MetaEntry->Offsets = CopyDims(DimCount, Offsets); @@ -776,17 +748,20 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, MetaEntry->BlockCount++; MetaEntry->Count = AppendDims(MetaEntry->Count, PreviousDBCount, DimCount, Count); - MetaEntry->DataLocation = - (size_t *)realloc(MetaEntry->DataLocation, + MetaEntry->DataBlockLocation = + (size_t *)realloc(MetaEntry->DataBlockLocation, MetaEntry->BlockCount * sizeof(size_t)); - MetaEntry->DataLocation[MetaEntry->BlockCount - 1] = DataOffset; + MetaEntry->DataBlockLocation[MetaEntry->BlockCount - 1] = + DataOffset; if (Rec->OperatorType) { MetaArrayRecOperator *OpEntry = (MetaArrayRecOperator *)MetaEntry; - OpEntry->DataLengths = (size_t *)realloc( - OpEntry->DataLengths, OpEntry->BlockCount * sizeof(size_t)); - OpEntry->DataLengths[OpEntry->BlockCount - 1] = CompressedSize; + OpEntry->DataBlockSize = + (size_t *)realloc(OpEntry->DataBlockSize, + OpEntry->BlockCount * sizeof(size_t)); + OpEntry->DataBlockSize[OpEntry->BlockCount - 1] = + CompressedSize; } if (m_StatsLevel > 0) { @@ -907,10 +882,33 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep, if (!Info.MetaFormat && Info.MetaFieldCount) { MetaMetaInfoBlock Block; - FMStructDescRec struct_list[4] = { + FMStructDescRec struct_list[20] = { {NULL, NULL, 0, NULL}, {"complex4", fcomplex_field_list, sizeof(fcomplex_struct), NULL}, {"complex8", dcomplex_field_list, sizeof(dcomplex_struct), NULL}, + {"MetaArray", MetaArrayRecListPtr, sizeof(MetaArrayRec), NULL}, + {"MetaArrayOp", MetaArrayRecOperatorListPtr, + sizeof(MetaArrayRecOperator), NULL}, + {"MetaArrayMM1", MetaArrayRecMM1ListPtr, sizeof(MetaArrayRecMM), + NULL}, + {"MetaArrayOpMM1", MetaArrayRecOperatorMM1ListPtr, + sizeof(MetaArrayRecOperatorMM), NULL}, + {"MetaArrayMM2", MetaArrayRecMM2ListPtr, sizeof(MetaArrayRecMM), + NULL}, + {"MetaArrayOperatorMM2", MetaArrayRecOperatorMM2ListPtr, + sizeof(MetaArrayRecOperatorMM), NULL}, + {"MetaArrayMM4", MetaArrayRecMM4ListPtr, sizeof(MetaArrayRecMM), + NULL}, + {"MetaArrayOpMM4", MetaArrayRecOperatorMM4ListPtr, + sizeof(MetaArrayRecOperatorMM), NULL}, + {"MetaArrayMM8", MetaArrayRecMM8ListPtr, sizeof(MetaArrayRecMM), + NULL}, + {"MetaArrayOpMM8", MetaArrayRecOperatorMM8ListPtr, + sizeof(MetaArrayRecOperatorMM), NULL}, + {"MetaArrayMM16", MetaArrayRecMM16ListPtr, sizeof(MetaArrayRecMM), + NULL}, + {"MetaArrayOpMM16", MetaArrayRecOperatorMM16ListPtr, + sizeof(MetaArrayRecOperatorMM), NULL}, {NULL, NULL, 0, NULL}}; struct_list[0].format_name = "MetaData"; struct_list[0].field_list = Info.MetaFields; @@ -989,7 +987,7 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep, new BufferFFS(AttributeEncodeBuffer, AttributeBlock, AttributeSize); } - // FMdump_encoded_data(Info.MetaFormat, MetaDataBlock, 1024000); + // FMdump_encoded_data(Info.MetaFormat, MetaDataBlock, 1024000); /* free all those copied dimensions, etc */ MBase = (struct BP5MetadataInfoStruct *)Metadata; size_t *tmp = MBase->BitField; From e01ba946db3e30df5399eb24a822ce7be0e5a8b2 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer <eisen@cc.gatech.edu> Date: Mon, 27 Jun 2022 10:20:04 -0400 Subject: [PATCH 2/6] kill assert, fix typename --- source/adios2/toolkit/format/bp5/BP5Deserializer.cpp | 2 +- source/adios2/toolkit/format/bp5/BP5Serializer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 9d10777857..3833a13a9d 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -174,7 +174,7 @@ void BP5Deserializer::BreakdownVarName(const char *Name, char **base_name_p, void BP5Deserializer::BreakdownFieldType(const char *FieldType, bool &Operator, bool &MinMax) { - assert(FieldType[0] == 'M'); // sanity check, should start with "MetaArray" + // should start with "MetaArray" FieldType += strlen("MetaArray"); if (FieldType == 0) return; diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index f3d0455035..267a1919b9 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -895,7 +895,7 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep, sizeof(MetaArrayRecOperatorMM), NULL}, {"MetaArrayMM2", MetaArrayRecMM2ListPtr, sizeof(MetaArrayRecMM), NULL}, - {"MetaArrayOperatorMM2", MetaArrayRecOperatorMM2ListPtr, + {"MetaArrayOpMM2", MetaArrayRecOperatorMM2ListPtr, sizeof(MetaArrayRecOperatorMM), NULL}, {"MetaArrayMM4", MetaArrayRecMM4ListPtr, sizeof(MetaArrayRecMM), NULL}, From 1f52680d22ab0a85206742533a3a1fd61a1217bc Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer <eisen@cc.gatech.edu> Date: Mon, 27 Jun 2022 10:32:22 -0400 Subject: [PATCH 3/6] Remove unused --- source/adios2/toolkit/format/bp5/BP5Serializer.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index 267a1919b9..331b825003 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -245,17 +245,6 @@ char *BP5Serializer::BuildVarName(const char *base_name, const ShapeID Shape, return Ret; } -static char *BuildShortName(const ShapeID Shape, const int Index, - const char *Suffix) -{ - - const char *Prefix = NamePrefix(Shape); - int Len = strlen(Prefix) + 2 + strlen(Suffix) + 16; - char *Ret = (char *)malloc(Len); - sprintf(Ret, "%s_%d_%s", Prefix, Index, Suffix); - return Ret; -} - static char *BuildLongName(const char *base_name, const ShapeID Shape, const int type, const int element_size) { From 46e5bcb23f0417fcc7055047969a1e59519b9e01 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer <eisen@cc.gatech.edu> Date: Mon, 27 Jun 2022 10:54:03 -0400 Subject: [PATCH 4/6] field overlap --- source/adios2/toolkit/format/bp5/BP5Base.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/toolkit/format/bp5/BP5Base.cpp b/source/adios2/toolkit/format/bp5/BP5Base.cpp index b74a1498a0..a84a001b78 100644 --- a/source/adios2/toolkit/format/bp5/BP5Base.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Base.cpp @@ -83,7 +83,7 @@ static FMField MetaArrayRecOperatorMM1List[] = { "DataBlockSize", "integer[BlockCount]", sizeof(size_t), FMOffset(BP5Base::MetaArrayRecOperator *, DataBlockSize)}, {"MinMax", "char[2][BlockCount]", 1, - FMOffset(BP5Base::MetaArrayRecMM *, MinMax)}, + FMOffset(BP5Base::MetaArrayRecOperatorMM *, MinMax)}, {NULL, NULL, 0, 0}}; static FMField MetaArrayRecMM2List[] = { BASE_FIELD_ENTRIES{"MinMax", "char[4][BlockCount]", 1, From 4b8415e55747858a1ee8f52495c8a382d801a29d Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer <eisen@cc.gatech.edu> Date: Mon, 27 Jun 2022 13:47:12 -0400 Subject: [PATCH 5/6] Throw an exception when reading old style metadata --- source/adios2/toolkit/format/bp5/BP5Deserializer.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 3833a13a9d..7639d5f55e 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -174,6 +174,12 @@ void BP5Deserializer::BreakdownVarName(const char *Name, char **base_name_p, void BP5Deserializer::BreakdownFieldType(const char *FieldType, bool &Operator, bool &MinMax) { + if (FieldType[0] != 'M') + { + throw std::runtime_error( + "BP5 unable to parse metadata, likely old version"); + } + // should start with "MetaArray" FieldType += strlen("MetaArray"); if (FieldType == 0) From 37ad960b019298ff72187b6592e7d05e99081332 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer <eisen@cc.gatech.edu> Date: Mon, 27 Jun 2022 20:40:30 -0400 Subject: [PATCH 6/6] Backwards compatible with prior BP5 metadata format --- .../toolkit/format/bp5/BP5Deserializer.cpp | 61 +++++++++++++++++-- .../toolkit/format/bp5/BP5Deserializer.h | 3 + 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 7639d5f55e..eb3c52786a 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -195,6 +195,40 @@ void BP5Deserializer::BreakdownFieldType(const char *FieldType, bool &Operator, } } +void BP5Deserializer::BreakdownV1ArrayName(const char *Name, char **base_name_p, + DataType *type_p, + int *element_size_p, bool &Operator, + bool &MinMax) +{ + int Type; + int ElementSize; + + const char *NameStart = strchr(strchr(Name + 4, '_') + 1, '_') + 1; + // + 3 to skip BP5_ or bp5_ prefix + sscanf(Name + 4, "%d_%d", &ElementSize, &Type); + const char *Plus = index(Name, '+'); + MinMax = false; + while (Plus && (*Plus == '+')) + { + int Len; + if (sscanf(Plus, "+%dO", &Len) == 1) + { // Operator Spec + Operator = true; + const char *OpStart = index(Plus, 'O') + 1; + Plus = OpStart + Len; + } + else if (strncmp(Plus, "+MM", 3) == 0) + { + MinMax = true; + Plus += 3; + } + } + *element_size_p = ElementSize; + *type_p = (DataType)Type; + *base_name_p = strdup(NameStart); + *(rindex(*base_name_p, '_')) = 0; +} + void BP5Deserializer::BreakdownArrayName(const char *Name, char **base_name_p, DataType *type_p, int *element_size_p) { @@ -287,9 +321,20 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) int ElementSize; bool Operator = false; bool MinMax = false; - BreakdownFieldType(FieldList[i].field_type, Operator, MinMax); - BreakdownArrayName(FieldList[i].field_name, &ArrayName, &Type, - &ElementSize); + bool V1_fields = true; + if (FieldList[i].field_type[0] == 'M') + V1_fields = false; + if (V1_fields) + { + BreakdownV1ArrayName(FieldList[i + 4].field_name, &ArrayName, + &Type, &ElementSize, Operator, MinMax); + } + else + { + BreakdownFieldType(FieldList[i].field_type, Operator, MinMax); + BreakdownArrayName(FieldList[i].field_name, &ArrayName, &Type, + &ElementSize); + } VarRec = LookupVarByName(ArrayName); if (!VarRec) { @@ -311,8 +356,16 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) { VarRec->MinMaxOffset = MetaRecFields * sizeof(void *); + MetaRecFields++; + } + if (V1_fields) + { + i += MetaRecFields; + } + else + { + i++; } - i++; free(ArrayName); } else diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.h b/source/adios2/toolkit/format/bp5/BP5Deserializer.h index 0e4365a99c..5f7a1db902 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.h @@ -186,6 +186,9 @@ class BP5Deserializer : virtual public BP5Base bool &MinMax); void BreakdownArrayName(const char *Name, char **base_name_p, DataType *type_p, int *element_size_p); + void BreakdownV1ArrayName(const char *Name, char **base_name_p, + DataType *type_p, int *element_size_p, + bool &Operator, bool &MinMax); void *VarSetup(core::Engine *engine, const char *variableName, const DataType type, void *data); void *ArrayVarSetup(core::Engine *engine, const char *variableName,