Skip to content

Commit

Permalink
Merge pull request #225 from adv12/ImportPerformance
Browse files Browse the repository at this point in the history 
Improve Import Performance
  • Loading branch information
@strhea
strhea authored Dec 2, 2024
2 parents 4bda9d2 + edd679e commit 005b266
Show file tree
Hide file tree
Showing 14 changed files with 286 additions and 81 deletions.
28 changes: 28 additions & 0 deletions ISOv4Plugin/ExtensionMethods/ExtensionMethods.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,34 @@ public static string AsHexDDI(this int n)
return n.ToString("X4");
}

public static bool ReverseEquals(this string s1, string s2)
{
if (ReferenceEquals(s1, s2))
{
return true;
}

if (s1 == null || s2 == null)
{
return false;
}

if (s1.Length != s2.Length)
{
return false;
}

for (int i = s1.Length - 1; i >= 0; i--)
{
if (s1[i] != s2[i])
{
return false;
}
}

return true;
}


/// <summary>
/// Looks up unit, converts and loads representation
Expand Down
13 changes: 11 additions & 2 deletions ISOv4Plugin/ISOModels/ISODataLogValue.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,17 @@ namespace AgGateway.ADAPT.ISOv4Plugin.ISOModels
{
public class ISODataLogValue : ISOElement
{
//Attributes
public string ProcessDataDDI { get; set; }
private string _processDataDDI;
public string ProcessDataDDI
{
get => _processDataDDI;
set
{
_processDataDDI = value;
ProcessDataIntDDI = value.AsInt32DDI();
}
}
public int ProcessDataIntDDI { get; private set; }
public int? ProcessDataValue { get; set; }
public string DeviceElementIdRef { get; set; }
public uint? DataLogPGN { get; set; }
Expand Down
30 changes: 30 additions & 0 deletions ISOv4Plugin/ISOModels/ISODevice.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,36 @@ public ISODevice()
public List<ISODeviceProperty> DeviceProperties { get; set; }
public List<ISODeviceValuePresentation> DeviceValuePresentations { get; set; }

//Performance optimizations
private Dictionary<int, ISODeviceProcessData> _deviceProcessDataByIntDDI;

public ISODeviceProcessData FirstOrDefaultDeviceProcessData(int intDDI)
{
if (DeviceProcessDatas == null)
{
return null;
}
if (_deviceProcessDataByIntDDI == null)
{
_deviceProcessDataByIntDDI = new Dictionary<int, ISODeviceProcessData>();
foreach (var dpd2 in DeviceProcessDatas)
{
if (!_deviceProcessDataByIntDDI.TryGetValue(dpd2.IntDDI, out _))
{
_deviceProcessDataByIntDDI[dpd2.IntDDI] = dpd2;
}
}
}

ISODeviceProcessData dpd;
if (_deviceProcessDataByIntDDI.TryGetValue(intDDI, out dpd))
{
return dpd;
}

return null;
}

public override XmlWriter WriteXML(XmlWriter xmlBuilder)
{
xmlBuilder.WriteStartElement("DVC");
Expand Down
12 changes: 11 additions & 1 deletion ISOv4Plugin/ISOModels/ISODeviceProcessData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,17 @@ public class ISODeviceProcessData : ISOElement
{
//Attributes
public uint ObjectID { get; set; }
public string DDI { get; set; }
private string _ddi;
public string DDI
{
get => _ddi;
set
{
_ddi = value;
IntDDI = value.AsInt32DDI();
}
}
public int IntDDI { get; private set; }
public int Property { get; set; }
public int TriggerMethods { get; set; }
public string Designator { get; set; }
Expand Down
2 changes: 1 addition & 1 deletion ISOv4Plugin/Mappers/Factories/TimeLogMapperFactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ private List<TimeLogWrapperGroup> HandleDuplicateDataLogValues(List<TimeLogWrapp
foreach (var timeLogGroup in timeLogGroups)
{
var duplicatesByElementAndDDI = timeLogGroup.SelectMany(x => x.DataLogValues)
.Where(x => x.DataLogPGN == null && x.ProcessDataDDI != "0000")
.Where(x => x.DataLogPGN == null && x.ProcessDataIntDDI != 0x0000)
.GroupBy(x => new { x.DeviceElementIdRef, x.ProcessDataDDI })
.Where(x => x.Count() > 1);
if (!timeLogGroup.KeepAsGroup && duplicatesByElementAndDDI.Any())
Expand Down
4 changes: 2 additions & 2 deletions ISOv4Plugin/Mappers/LoggedDataMappers/Import/CondensedWorkStateMeterCreator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,13 +106,13 @@ public abstract class CondensedStateMeterCreator : IEnumeratedMeterCreator
public List<ISOEnumeratedMeter> CreateMeters(IEnumerable<ISOSpatialRow> spatialRows, ISODataLogValue dlv)
{
//We need to find a row of data with the value in order to create the correct number of meters.
var spatialRowWithDdi = spatialRows.FirstOrDefault(x => x.SpatialValues.Any(y => y.DataLogValue.ProcessDataDDI.AsInt32DDI() == DDI
var spatialRowWithDdi = spatialRows.FirstOrDefault(x => x.SpatialValues.Any(y => y.DataLogValue.ProcessDataIntDDI == DDI
&& y.DataLogValue.DeviceElementIdRef == dlv.DeviceElementIdRef));

int numberOfSections = 0;
if (spatialRowWithDdi != null)
{
var spatialValue = spatialRowWithDdi.SpatialValues.First(x => x.DataLogValue.ProcessDataDDI.AsInt32DDI() == DDI &&
var spatialValue = spatialRowWithDdi.SpatialValues.First(x => x.DataLogValue.ProcessDataIntDDI == DDI &&
x.DataLogValue.DeviceElementIdRef == dlv.DeviceElementIdRef);
numberOfSections = GetNumberOfInstalledSections(spatialValue);
}
Expand Down
74 changes: 52 additions & 22 deletions ISOv4Plugin/Mappers/LoggedDataMappers/Import/SectionMapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Linq;
using AgGateway.ADAPT.ApplicationDataModel.Equipment;
using AgGateway.ADAPT.ApplicationDataModel.LoggedData;
using AgGateway.ADAPT.ISOv4Plugin.ExtensionMethods;
using AgGateway.ADAPT.ISOv4Plugin.ISOModels;
using AgGateway.ADAPT.ISOv4Plugin.ObjectModel;

Expand Down Expand Up @@ -30,17 +31,50 @@ public List<DeviceElementUse> Map(ISOTime time,
IEnumerable<string> isoDeviceElementIDs,
Dictionary<string, List<ISOProductAllocation>> isoProductAllocations)
{
var usedDataLogValues = new List<ISODataLogValue>();

foreach (string isoDeviceElementID in isoDeviceElementIDs)
{
DeviceHierarchyElement hierarchyElement = TaskDataMapper.DeviceElementHierarchies.GetMatchingElement(isoDeviceElementID);
int? adaptDeviceElementId = TaskDataMapper.InstanceIDMap.GetADAPTID(isoDeviceElementID);
if (hierarchyElement != null &&
adaptDeviceElementId.HasValue &&
DataModel.Catalog.DeviceElements.Any(d => d.Id.ReferenceId == adaptDeviceElementId.Value))
{
usedDataLogValues.AddRange(_workingDataMapper.GetDataLogValuesForDeviceElement(time, hierarchyElement));
}
}

List<ISOSpatialRow> isoRecordsWithData = new List<ISOSpatialRow>();
if (usedDataLogValues.Any())
{
foreach (var isoRecord in isoRecords)
{
int beforeCount = usedDataLogValues.Count;
var notReferencedDataLogValues = usedDataLogValues.Where(x =>
!isoRecord.SpatialValues.Any(y => y.DataLogValue.ProcessDataIntDDI == x.ProcessDataIntDDI &&
y.DataLogValue.DeviceElementIdRef.ReverseEquals(x.DeviceElementIdRef)));
usedDataLogValues = notReferencedDataLogValues.ToList();
if (beforeCount != usedDataLogValues.Count)
{
isoRecordsWithData.Add(isoRecord);
}
if (usedDataLogValues.Count == 0)
{
break;
}
}
}


var sections = new List<DeviceElementUse>();
foreach (string isoDeviceElementID in isoDeviceElementIDs)
{
DeviceHierarchyElement hierarchyElement = TaskDataMapper.DeviceElementHierarchies.GetMatchingElement(isoDeviceElementID);
if (hierarchyElement != null)
{
DeviceElementUse deviceElementUse = null;
List<WorkingData> workingDatas = new List<WorkingData>();

//Get the relevant DeviceElementConfiguration
int adaptDeviceElementId = TaskDataMapper.InstanceIDMap.GetADAPTID(isoDeviceElementID).Value;
int? adaptDeviceElementId = TaskDataMapper.InstanceIDMap.GetADAPTID(isoDeviceElementID);
DeviceElement adaptDeviceElement = DataModel.Catalog.DeviceElements.SingleOrDefault(d => d.Id.ReferenceId == adaptDeviceElementId);
if (adaptDeviceElement != null)
{
Expand All @@ -55,33 +89,29 @@ public List<DeviceElementUse> Map(ISOTime time,
order = hierarchyElement.Parent.Order;
}

deviceElementUse = sections.FirstOrDefault(d => d.DeviceConfigurationId == config.Id.ReferenceId);
List<WorkingData> workingDatas = new List<WorkingData>();
DeviceElementUse deviceElementUse = sections.FirstOrDefault(d => d.DeviceConfigurationId == config.Id.ReferenceId);
if (deviceElementUse == null)
{
//Create the DeviceElementUse
deviceElementUse = new DeviceElementUse();
deviceElementUse.Depth = depth;
deviceElementUse.Order = order;
deviceElementUse.OperationDataId = operationDataId;
deviceElementUse.DeviceConfigurationId = config.Id.ReferenceId;

//Add Working Data for any data on this device element
List<WorkingData> data = _workingDataMapper.Map(time, isoRecords, deviceElementUse, hierarchyElement, sections, isoProductAllocations);
if (data.Any())
deviceElementUse = new DeviceElementUse
{
workingDatas.AddRange(data);
}
Depth = depth,
Order = order,
OperationDataId = operationDataId,
DeviceConfigurationId = config.Id.ReferenceId
};
}
else
{
workingDatas = deviceElementUse.GetWorkingDatas().ToList();
}

//Add Additional Working Data
List<WorkingData> data = _workingDataMapper.Map(time, isoRecords, deviceElementUse, hierarchyElement, sections, isoProductAllocations);
if (data.Any())
{
workingDatas.AddRange(data);
}
//Add Working Data for any data on this device element
List<WorkingData> data = _workingDataMapper.Map(time, isoRecordsWithData, deviceElementUse, hierarchyElement, sections, isoProductAllocations);
if (data.Any())
{
workingDatas.AddRange(data);
}

deviceElementUse.GetWorkingDatas = () => workingDatas;
Expand Down
2 changes: 1 addition & 1 deletion ISOv4Plugin/Mappers/LoggedDataMappers/Import/SkyConditionsMeterCreator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public List<ISOEnumeratedMeter> CreateMeters(IEnumerable<ISOSpatialRow> spatialR

public EnumeratedValue GetValueForMeter(SpatialValue value, ISOEnumeratedMeter meter)
{
if (value.DataLogValue.ProcessDataDDI.AsInt32DDI() != DDI)
if (value.DataLogValue.ProcessDataIntDDI != DDI)
return null;

const int clear = 0x20524C43;
Expand Down
17 changes: 9 additions & 8 deletions ISOv4Plugin/Mappers/LoggedDataMappers/Import/SpatialRecordMapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using AgGateway.ADAPT.ApplicationDataModel.LoggedData;
using AgGateway.ADAPT.ApplicationDataModel.Representations;
using AgGateway.ADAPT.ISOv4Plugin.ExtensionMethods;
using AgGateway.ADAPT.ISOv4Plugin.ObjectModel;
using AgGateway.ADAPT.ISOv4Plugin.ISOModels;
using AgGateway.ADAPT.Representation.UnitSystem;
Expand Down Expand Up @@ -91,8 +92,9 @@ private void SetEnumeratedMeterValue(ISOSpatialRow isoSpatialRow, EnumeratedWork
{
var isoDataLogValue = _workingDataMapper.DataLogValuesByWorkingDataID[meter.Id.ReferenceId];
var isoValue = isoSpatialRow.SpatialValues.FirstOrDefault(v =>
v.DataLogValue.DeviceElementIdRef == isoDataLogValue.DeviceElementIdRef &&
v.DataLogValue.ProcessDataDDI == isoDataLogValue.ProcessDataDDI);
v.DataLogValue.ProcessDataIntDDI == isoDataLogValue.ProcessDataIntDDI &&
v.DataLogValue.DeviceElementIdRef.ReverseEquals(isoDataLogValue.DeviceElementIdRef)
);
if (isoValue != null)
{
var isoEnumeratedMeter = meter as ISOEnumeratedMeter;
Expand All @@ -112,14 +114,13 @@ private void SetNumericMeterValue(ISOSpatialRow isoSpatialRow, NumericWorkingDat
var dataLogValue = _workingDataMapper.DataLogValuesByWorkingDataID.ContainsKey(meter.Id.ReferenceId)
? _workingDataMapper.DataLogValuesByWorkingDataID[meter.Id.ReferenceId]
: null;
var isoValue = dataLogValue != null
var isoValue = dataLogValue != null && dataLogValue.ProcessDataIntDDI != 0xDFEE
? isoSpatialRow.SpatialValues.FirstOrDefault(v =>
v.DataLogValue.ProcessDataDDI != "DFFE" &&
v.DataLogValue.DeviceElementIdRef == dataLogValue.DeviceElementIdRef &&
v.DataLogValue.ProcessDataDDI == dataLogValue.ProcessDataDDI)
v.DataLogValue.ProcessDataIntDDI == dataLogValue.ProcessDataIntDDI &&
v.DataLogValue.DeviceElementIdRef.ReverseEquals(dataLogValue.DeviceElementIdRef)
)
: null;


if (isoValue != null)
{
ADAPT.ApplicationDataModel.Common.UnitOfMeasure userProvidedUnitOfMeasure = meter.UnitOfMeasure; //Default; no display uom provided.
Expand Down Expand Up @@ -218,7 +219,7 @@ private DateTime ToUtc(DateTime dateTime)
}
else if (dateTime.Kind == DateTimeKind.Unspecified && _taskDataMapper.GPSToLocalDelta.HasValue)
{
utc = new DateTime(dateTime.AddHours(- _taskDataMapper.GPSToLocalDelta.Value).Ticks, DateTimeKind.Utc);
utc = new DateTime(dateTime.AddHours(-_taskDataMapper.GPSToLocalDelta.Value).Ticks, DateTimeKind.Utc);
}
else
{
Expand Down
19 changes: 14 additions & 5 deletions ISOv4Plugin/Mappers/LoggedDataMappers/Import/WorkingDataMapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ public interface IWorkingDataMapper
WorkingData ConvertToBaseType(WorkingData meter);
Dictionary<int, ISODataLogValue> DataLogValuesByWorkingDataID { get; set; }
Dictionary<int, string> ISODeviceElementIDsByWorkingDataID { get; set; }

IEnumerable<ISODataLogValue> GetDataLogValuesForDeviceElement(ISOTime time, DeviceHierarchyElement deviceElementHierarchy);
}

public class WorkingDataMapper : BaseMapper, IWorkingDataMapper
Expand All @@ -40,6 +42,13 @@ public WorkingDataMapper(IEnumeratedMeterFactory enumeratedMeterCreatorFactory,
ISODeviceElementIDsByWorkingDataID = new Dictionary<int, string>();
}

public IEnumerable<ISODataLogValue> GetDataLogValuesForDeviceElement(ISOTime time, DeviceHierarchyElement deviceElementHierarchy)
{
return time.DataLogValues.Where(dlv => dlv.DeviceElementIdRef == deviceElementHierarchy.DeviceElement.DeviceElementId || //DLV DET reference matches the primary DET for the ADAPT element
deviceElementHierarchy.MergedElements.Any(e => e.DeviceElementId == dlv.DeviceElementIdRef)) //DLV DET reference matches one of the merged DETs on the ADAPT element
.ToList();
}

public List<WorkingData> Map(ISOTime time,
IEnumerable<ISOSpatialRow> isoSpatialRows,
DeviceElementUse deviceElementUse,
Expand Down Expand Up @@ -73,7 +82,7 @@ public List<WorkingData> Map(ISOTime time,
isoDeviceElementHierarchy);
if (newWorkingDatas.Count() > 0)
{
int ddi = dlv.ProcessDataDDI.AsInt32DDI();
int ddi = dlv.ProcessDataIntDDI;
if (!EnumeratedMeterFactory.IsCondensedMeter(ddi))
{
//We skip adding Condensed WorkingDatas to this DeviceElementUse since they were added separately below to their specific DeviceElementUse
Expand Down Expand Up @@ -126,7 +135,7 @@ private IEnumerable<WorkingData> Map(ISODataLogValue dlv,
DeviceHierarchyElement isoDeviceElementHierarchy)
{
var workingDatas = new List<WorkingData>();
if (_ddis.ContainsKey(dlv.ProcessDataDDI.AsInt32DDI()))
if (_ddis.ContainsKey(dlv.ProcessDataIntDDI))
{
//Numeric Representations
NumericWorkingData numericMeter = MapNumericMeter(dlv, deviceElementUse.Id.ReferenceId);
Expand All @@ -135,7 +144,7 @@ private IEnumerable<WorkingData> Map(ISODataLogValue dlv,
workingDatas.Add(numericMeter);
return workingDatas;
}
var meterCreator = _enumeratedMeterCreatorFactory.GetMeterCreator(dlv.ProcessDataDDI.AsInt32DDI());
var meterCreator = _enumeratedMeterCreatorFactory.GetMeterCreator(dlv.ProcessDataIntDDI);
if (meterCreator != null)
{
//Enumerated Representations
Expand Down Expand Up @@ -187,9 +196,9 @@ private NumericWorkingData MapNumericMeter(ISODataLogValue dlv, int deviceElemen
{
var meter = new NumericWorkingData
{
UnitOfMeasure = RepresentationMapper.GetUnitForDdi(dlv.ProcessDataDDI.AsInt32DDI()),
UnitOfMeasure = RepresentationMapper.GetUnitForDdi(dlv.ProcessDataIntDDI),
DeviceElementUseId = deviceElementUseId,
Representation = RepresentationMapper.Map(dlv.ProcessDataDDI.AsInt32DDI())
Representation = RepresentationMapper.Map(dlv.ProcessDataIntDDI)
};
return meter;
}
Expand Down
1 change: 1 addition & 0 deletions ISOv4Plugin/Mappers/TaskDataMapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,7 @@ public ApplicationDataModel.ADM.ApplicationDataModel Import(ISO11783_TaskData ta
//During DET import, we use the DeviceElementHierarchies from above to map the actual hierarchies and fill in details.
DeviceMapper deviceMapper = new DeviceMapper(this);
AdaptDataModel.Catalog.DeviceModels.AddRange(deviceMapper.ImportDevices(devices));
DeviceElementHierarchies.CacheDeviceElementIds();
}

//Workers
Expand Down
Loading

0 comments on commit 005b266

Please sign in to comment.