Skip to content

Commit 6465a20

Browse files
MichaelHuthMichaelHuth
committed
NWB: Recreate Epochs on export and insert this info into LBN
- Epochs for DA channels get recreated. - Epoch recreation is an optional argument for NWB_ExportAllData and is by default off. - The location where the LNB is written to the NWB file through NWB_WriteLabnotebooksAndComments was moved to a spot after the LNB was adapted. Added tests for postProcessed epoch addition for vintage files.
1 parent 59e1a16 commit 6465a20

File tree

5 files changed

+207
-5
lines changed

5 files changed

+207
-5
lines changed

‎Packages/MIES/MIES_Epochs.ipf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1741,7 +1741,7 @@ End
17411741
/// @param sweepDFR single sweep folder, e.g. for measurement with a device this wold be DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo)
17421742
/// @param sweepNo sweep number
17431743
/// @returns recreated 4D epoch wave
1744-
static Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo)
1744+
Function/WAVE EP_RecreateEpochsFromLoadedData(WAVE numericalValues, WAVE/T textualValues, DFREF sweepDFR, variable sweepNo)
17451745

17461746
STRUCT DataConfigurationResult s
17471747
variable channelNr, plannedTime, acquiredTime, adSize, firstUnacquiredIndex

‎Packages/MIES/MIES_MiesUtilities_Logbook.ipf‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1977,3 +1977,66 @@ Function ParseLogbookMode(string modeText)
19771977
break
19781978
endswitch
19791979
End
1980+
1981+
Function InsertRecreatedEpochsIntoLBN(WAVE numericalValues, WAVE/T textualValues, string device, variable sweepNo)
1982+
1983+
string epochList
1984+
variable channelNumber, channelType, headstage, numChannelTypes, colCount, allocatedCols
1985+
variable assocCol = NaN
1986+
1987+
DFREF deviceDFR = GetDeviceDataPath(device)
1988+
DFREF sweepDFR = GetSingleSweepFolder(deviceDFR, sweepNo)
1989+
WAVE/Z recEpochs = EP_RecreateEpochsFromLoadedData(numericalValues, textualValues, sweepDFR, sweepNo)
1990+
if(!WaveExists(recEpochs))
1991+
print "Could not recreate Epochs."
1992+
return NaN
1993+
endif
1994+
1995+
Make/FREE channelTypes = {XOP_CHANNEL_TYPE_DAC, XOP_CHANNEL_TYPE_TTL}
1996+
1997+
numChannelTypes = DimSize(channelTypes, ROWS)
1998+
allocatedCols = numChannelTypes * NUM_DA_TTL_CHANNELS
1999+
Make/FREE/T/N=(1, allocatedCols) keys
2000+
Make/FREE/T/N=(1, allocatedCols, LABNOTEBOOK_LAYER_COUNT) values
2001+
2002+
for(channelType : channelTypes)
2003+
for(channelNumber = 0; channelNumber < NUM_DA_TTL_CHANNELS; channelNumber += 1)
2004+
// Currently only implemented for DAC channel type
2005+
if(channelType != XOP_CHANNEL_TYPE_DAC)
2006+
continue
2007+
endif
2008+
2009+
epochList = EP_EpochWaveToStr(recEpochs, channelNumber, channelType)
2010+
if(IsEmpty(epochList))
2011+
continue
2012+
endif
2013+
2014+
headstage = GetHeadstageFromLBN(numericalValues, sweepNo, channelType, channelNumber, DATA_ACQUISITION_MODE)
2015+
if(IsAssociatedChannel(headstage))
2016+
if(IsNaN(assocCol))
2017+
assocCol = colCount
2018+
colCount += 1
2019+
keys[0][assocCol] = EPOCHS_ENTRY_KEY
2020+
endif
2021+
values[0][assocCol][headstage] = epochList
2022+
continue
2023+
endif
2024+
2025+
values[0][colCount][INDEP_HEADSTAGE] = epochList
2026+
keys[0][colCount] = CreateLBNUnassocKey(EPOCHS_ENTRY_KEY, channelNumber, channelType)
2027+
colCount += 1
2028+
endfor
2029+
endfor
2030+
if(colCount)
2031+
Redimension/N=(-1, colCount) keys
2032+
Redimension/N=(-1, colCount, -1) values
2033+
ED_AddEntriesToLabnotebook(values, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1)
2034+
endif
2035+
2036+
Redimension/N=(-1, 1) keys
2037+
keys[0][0] = SWEEP_EPOCH_VERSION_ENTRY_KEY
2038+
Make/FREE/D/N=(1, 1, LABNOTEBOOK_LAYER_COUNT) valuesNum
2039+
FastOp valuesNum = (NaN)
2040+
valuesNum[0][0][INDEP_HEADSTAGE] = SWEEP_EPOCH_VERSION
2041+
ED_AddEntriesToLabnotebook(valuesNum, keys, sweepNo, device, DATA_ACQUISITION_MODE, insertAsPostProc = 1)
2042+
End

‎Packages/MIES/MIES_NeuroDataWithoutBorders.ipf‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,6 @@ static Function NWB_AddDeviceSpecificData(STRUCT NWBAsyncParameters &s, variable
467467

468468
AddModificationTimeEntry(s.locationID, s.nwbVersion)
469469
NWB_AddDevice(s)
470-
NWB_WriteLabnotebooksAndComments(s)
471470
NWB_WriteTestpulseData(s, writeStoredTestPulses)
472471
End
473472

@@ -576,12 +575,13 @@ End
576575
/// @param overwrite [optional, defaults to false] overwrite any existing NWB file with the same name, only
577576
/// used when overrideFilePath is passed
578577
/// @param verbose [optional, defaults to true] get diagnostic output to the command line
578+
/// @param recreateEpochs [optional, defaults to false] when set to true, epoch information is recreated if the version of the data in the experiment is older than the latest version
579579
///
580580
/// @return 0 on success, non-zero on failure
581-
Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideFullFilePath, string overrideFileTemplate, variable writeStoredTestPulses, variable writeIgorHistory, variable compressionMode, variable keepFileOpen, variable overwrite, variable verbose])
581+
Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideFullFilePath, string overrideFileTemplate, variable writeStoredTestPulses, variable writeIgorHistory, variable compressionMode, variable keepFileOpen, variable overwrite, variable verbose, variable recreateEpochs])
582582

583583
string list, name, fileName
584-
variable locationID, sweep, createdNewNWBFile, argCheck
584+
variable locationID, sweep, createdNewNWBFile, argCheck, epochVersion
585585
string stimsetList = ""
586586

587587
if(ParamIsDefault(keepFileOpen))
@@ -618,6 +618,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF
618618
verbose = !!verbose
619619
endif
620620

621+
recreateEpochs = ParamIsDefault(recreateEpochs) ? 0 : !!recreateEpochs
622+
621623
argCheck = ParamIsDefault(overrideFullFilePath) + ParamIsDefault(overrideFileTemplate)
622624
ASSERT(argCheck >= 1 && argCheck <= 2, "Either arg overrideFullFilePath or arg overrideFileTemplate must be given or none (auto-gen)")
623625

@@ -724,6 +726,13 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF
724726
WAVE s.DAQDataWave = TextSweepToWaveRef(sweepWave)
725727
WAVE s.DAQConfigWave = configWave
726728

729+
if(recreateEpochs)
730+
epochVersion = GetLastSettingIndep(s.numericalValues, s.sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, DATA_ACQUISITION_MODE, defValue = NaN)
731+
if(IsNaN(epochVersion) || epochVersion < SWEEP_EPOCH_VERSION)
732+
InsertRecreatedEpochsIntoLBN(s.numericalValues, s.textualValues, s.device, s.sweep)
733+
endif
734+
endif
735+
727736
NWB_AppendSweepLowLevel(s)
728737
stimsetList += AB_GetStimsetFromPanel(device, sweep)
729738

@@ -732,6 +741,8 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF
732741
endfor
733742
LOG_AddEntry(PACKAGE_MIES, "export", keys = {"size [MiB]"}, values = {num2str(NWB_GetExportedFileSize(device))})
734743

744+
NWB_WriteLabnotebooksAndComments(s)
745+
735746
NWB_AppendStimset(nwbVersion, s.locationID, stimsetList, compressionMode)
736747

737748
if(writeIgorHistory)

‎Packages/tests/HistoricData/UTF_AttemptNWB2ExportOnOldData.ipf‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,14 @@ static Function TestExportingDataToNWB([string str])
1111

1212
LoadMIESFolderFromPXP("input:" + str)
1313

14+
WAVE devEpochVersionPre = GatherEpochVersions()
15+
1416
PathInfo home
1517
templateName = S_path + GetBaseName(str)
1618
// attempt export
17-
NWB_ExportAllData(nwbVersion, overrideFileTemplate = templateName, writeStoredTestPulses = 1, writeIgorHistory = 1)
19+
NWB_ExportAllData(nwbVersion, overrideFileTemplate = templateName, writeStoredTestPulses = 1, writeIgorHistory = 1, recreateEpochs = 1)
20+
21+
CheckIfPostProcessedEpochsAreAdded(devEpochVersionPre)
1822

1923
CHECK_NO_RTE()
2024

‎Packages/tests/UTF_HelperFunctions.ipf‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1740,3 +1740,127 @@ threadsafe static Function ParseConstantValues_Impl(string entry)
17401740

17411741
return str2num(str)
17421742
End
1743+
1744+
Function/WAVE GatherEpochVersions()
1745+
1746+
variable i
1747+
1748+
WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";")
1749+
Make/FREE/WAVE/N=(DimSize(devicesWithContent, ROWS)) devEpochVersion
1750+
for(device : devicesWithContent)
1751+
WAVE numericalValues = GetLBNumericalValues(device)
1752+
DFREF dfr = GetDeviceDataPath(device)
1753+
1754+
WAVE/T sweepWaveNames = ListToTextWave(GetListOfObjects(dfr, DATA_SWEEP_REGEXP), ";")
1755+
Make/FREE/N=(DimSize(sweepWaveNames, ROWS)) sweepNums
1756+
sweepNums[] = ExtractSweepNumber(sweepWaveNames[p])
1757+
Make/FREE/D/N=(WaveMax(sweepNums) + 1) epochVersion
1758+
for(sweep : sweepNums)
1759+
epochVersion[sweep] = GetLastSettingIndep(numericalValues, sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, DATA_ACQUISITION_MODE, defValue = NaN)
1760+
endfor
1761+
devEpochVersion[i] = epochVersion
1762+
i += 1
1763+
endfor
1764+
1765+
return devEpochVersion
1766+
End
1767+
1768+
Function FindLastLBNRow(WAVE values, variable sweep, string key, variable headstage, [variable val, string str, variable findAny])
1769+
1770+
variable sweepCol, row, i, keyCol
1771+
variable firstValue, lastValue
1772+
1773+
findAny = ParamIsDefault(findAny) ? 0 : !!findAny
1774+
if(!findAny)
1775+
REQUIRE_EQUAL_VAR(ParamIsDefault(val) + ParamIsDefault(str), 1)
1776+
endif
1777+
1778+
sweepCol = GetSweepColumn(values)
1779+
FindRange(values, sweepCol, sweep, DATA_ACQUISITION_MODE, firstValue, lastValue)
1780+
REQUIRE_NEQ_VAR(firstValue, NaN)
1781+
REQUIRE_NEQ_VAR(lastValue, NaN)
1782+
REQUIRE_LE_VAR(firstValue, lastValue)
1783+
keyCol = FindDimLabel(values, COLS, key)
1784+
CHECK_NEQ_VAR(keyCol, -2)
1785+
row = NaN
1786+
if(IsTextWave(values))
1787+
WAVE/T valuesT = values
1788+
for(i = lastValue; i >= firstValue; i -= 1)
1789+
if(findAny)
1790+
if(!IsEmpty(valuesT[i][keyCol][headstage]))
1791+
row = i
1792+
break
1793+
endif
1794+
else
1795+
if(!CmpStr(valuesT[i][keyCol][headstage], str))
1796+
row = i
1797+
break
1798+
endif
1799+
endif
1800+
endfor
1801+
else
1802+
for(i = lastValue; i >= firstValue; i -= 1)
1803+
if(findAny)
1804+
if(!IsNaN(values[i][keyCol][headstage]))
1805+
row = i
1806+
break
1807+
endif
1808+
else
1809+
if(values[i][keyCol][headstage] == val)
1810+
row = i
1811+
break
1812+
endif
1813+
endif
1814+
endfor
1815+
endif
1816+
1817+
return row
1818+
End
1819+
1820+
Function CheckIfPostProcessedEpochsAreAdded(WAVE/WAVE devEpochVersionPre)
1821+
1822+
variable devCnt, sweep, keyCol, row, i
1823+
1824+
WAVE/WAVE devEpochVersionAfter = GatherEpochVersions()
1825+
1826+
WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";")
1827+
for(device : devicesWithContent)
1828+
// check for postProcessed epoch info
1829+
WAVE numericalValues = GetLBNumericalValues(device)
1830+
WAVE/T textualValues = GetLBTextualValues(device)
1831+
1832+
DFREF dfr = GetDeviceDataPath(device)
1833+
WAVE/T sweepWaveNames = ListToTextWave(GetListOfObjects(dfr, DATA_SWEEP_REGEXP), ";")
1834+
1835+
WAVE epochVersionsPre = devEpochVersionPre[devCnt]
1836+
WAVE epochVersionsAfter = devEpochVersionAfter[devCnt]
1837+
1838+
for(name : sweepWaveNames)
1839+
sweep = ExtractSweepNumber(name)
1840+
if(epochVersionsPre[sweep] < SWEEP_EPOCH_VERSION)
1841+
CHECK_EQUAL_VAR(SWEEP_EPOCH_VERSION, epochVersionsAfter[sweep])
1842+
1843+
row = FindLastLBNRow(numericalValues, sweep, SWEEP_EPOCH_VERSION_ENTRY_KEY, INDEP_HEADSTAGE, val = SWEEP_EPOCH_VERSION)
1844+
CHECK_NEQ_VAR(row, NaN)
1845+
1846+
keyCol = FindDimLabel(numericalValues, COLS, POSTPROCESSED_ENTRY_KEY)
1847+
CHECK_NEQ_VAR(keyCol, -2)
1848+
CHECK_EQUAL_VAR(numericalValues[row][keyCol][INDEP_HEADSTAGE], 1)
1849+
1850+
for(i = 0; i < NUM_HEADSTAGES; i += 1)
1851+
row = FindLastLBNRow(textualValues, sweep, EPOCHS_ENTRY_KEY, i, findAny = 1)
1852+
if(!IsNaN(row))
1853+
break
1854+
endif
1855+
endfor
1856+
CHECK_NEQ_VAR(row, NaN)
1857+
1858+
keyCol = FindDimLabel(textualValues, COLS, POSTPROCESSED_ENTRY_KEY)
1859+
CHECK_NEQ_VAR(keyCol, -2)
1860+
CHECK_EQUAL_STR(textualValues[row][keyCol][INDEP_HEADSTAGE], "1")
1861+
endif
1862+
1863+
endfor
1864+
devCnt += 1
1865+
endfor
1866+
End

0 commit comments

Comments
 (0)