Skip to content

Commit

Permalink
Merge pull request #83 from usdot-jpo-ode/decoder
Browse files Browse the repository at this point in the history
Decoder
  • Loading branch information
John-Wiens authored May 31, 2024
2 parents cef5079 + f177ff9 commit b66b67d
Show file tree
Hide file tree
Showing 47 changed files with 2,210 additions and 38 deletions.
102 changes: 91 additions & 11 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,54 @@
FROM maven:3.8-eclipse-temurin-21-alpine as builder
# === BUILDER IMAGE for ACM ===
# FROM alpine:3.12 as builder
FROM amazonlinux:2023 as builder
USER root
WORKDIR /asn1_codec
## add build dependencies
RUN yum install -y cmake g++ make bash automake libtool autoconf flex bison

# Install librdkafka from Confluent repo
RUN rpm --import https://packages.confluent.io/rpm/7.6/archive.key
COPY ./confluent.repo /etc/yum.repos.d
RUN yum clean all
RUN yum install -y librdkafka-devel

# Install pugixml
ADD ./asn1_codec/pugixml /asn1_codec/pugixml
RUN cd /asn1_codec/pugixml && mkdir -p build && cd build && cmake .. && make && make install

# Build and install asn1c submodule
ADD ./asn1_codec/asn1c /asn1_codec/asn1c
RUN cd asn1c && test -f configure || autoreconf -iv && ./configure && make && make install

# Make generated files available to the build & compile example
RUN export LD_LIBRARY_PATH=/usr/local/lib
ADD ./asn1_codec/asn1c_combined /asn1_codec/asn1c_combined
RUN cd /asn1_codec/asn1c_combined && bash doIt.sh

# Remove any lingering .asn files
RUN rm -rf /asn1c_codec/asn1c_combined/j2735-asn-files
RUN rm -rf /asn1c_codec/asn1c_combined/semi-asn-files

# Remove duplicate files
RUN rm -rf /asn1c_codec/asn1c_combined/generated-files

# add the source and build files
ADD ./asn1_codec/CMakeLists.txt /asn1_codec
ADD ./asn1_codec/config /asn1_codec/config
ADD ./asn1_codec/include /asn1_codec/include
ADD ./asn1_codec/src /asn1_codec/src
ADD ./asn1_codec/kafka-test /asn1_codec/kafka-test
ADD ./asn1_codec/unit-test-data /asn1_codec/unit-test-data
ADD ./asn1_codec/data /asn1_codec/data
ADD ./asn1_codec/run_acm.sh /asn1_codec
ADD ./asn1_codec/data /asn1_codec/data

# Build acm.
RUN mkdir -p /build && cd /build && cmake /asn1_codec && make


# # === Build image for Java ===
FROM maven:3.8-eclipse-temurin-21-alpine as jbuilder

WORKDIR /home

Expand Down Expand Up @@ -37,17 +87,49 @@ RUN mvn clean install -DskipTests
WORKDIR /home/jpo-conflictvisualizer-api

RUN mvn clean package -DskipTests
# ENTRYPOINT ["tail", "-f", "/dev/null"]
FROM eclipse-temurin:21-jre-alpine
ENTRYPOINT ["tail", "-f", "/dev/null"]


# # === RUNTIME IMAGE for Java and ACM ===
# # Use Amazon Corretto Java on Amazon Linum 2023 to match the codec build env
# # FROM alpine:3.12
FROM amazoncorretto:21-al2023

WORKDIR /home

COPY --from=builder /home/jpo-conflictvisualizer-api/src/main/resources/application.yaml /home
COPY --from=builder /home/jpo-conflictvisualizer-api/src/main/resources/logback.xml /home
COPY --from=builder /home/jpo-conflictvisualizer-api/target/jpo-conflictvisualizer-api-1.3.0-SNAPSHOT.jar /home
COPY --from=jbuilder /home/jpo-conflictvisualizer-api/src/main/resources/application.yaml /home
COPY --from=jbuilder /home/jpo-conflictvisualizer-api/src/main/resources/logback.xml /home
COPY --from=jbuilder /home/jpo-conflictvisualizer-api/target/jpo-conflictvisualizer-api.jar /home
# # Copy asn1_codec executable and test files
USER root
WORKDIR /asn1_codec



# # add runtime dependencies
RUN yum install -y bash


# Install librdkafka from Confluent repo
RUN rpm --import https://packages.confluent.io/rpm/7.6/archive.key
COPY ./confluent.repo /etc/yum.repos.d
RUN yum clean all
RUN yum install -y librdkafka-devel

#COPY cert.crt /home/cert.crt
#RUN keytool -import -trustcacerts -keystore /opt/java/openjdk/lib/security/cacerts -storepass changeit -noprompt -alias mycert -file cert.crt
# # copy the built files from the builder
COPY --from=builder /asn1_codec /asn1_codec
COPY --from=builder /build /build

# # Add test data. This changes frequently so keep it low in the file.
ADD ./asn1_codec/docker-test /asn1_codec/docker-test

# # Put workdir back to Java home
WORKDIR /home


# # ENTRYPOINT ["java", "-jar", "/home/asn1-codec-java.jar"]
# # #COPY cert.crt /home/cert.crt
# # #RUN keytool -import -trustcacerts -keystore /opt/java/openjdk/lib/security/cacerts -storepass changeit -noprompt -alias mycert -file cert.crt

ENTRYPOINT ["java", \
"-Djava.rmi.server.hostname=$DOCKER_HOST_IP", \
Expand All @@ -59,6 +141,4 @@ ENTRYPOINT ["java", \
"-Dcom.sun.management.jmxremote.ssl=false", \
"-Dlogback.configurationFile=/home/logback.xml", \
"-jar", \
"/home/jpo-conflictvisualizer-api-0.0.1-SNAPSHOT.jar"]

# ENTRYPOINT ["tail", "-f", "/dev/null"]
"/home/jpo-conflictvisualizer-api.jar"]
1 change: 1 addition & 0 deletions asn1_codec
Submodule asn1_codec added at 422bca
13 changes: 13 additions & 0 deletions confluent.repo
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[Confluent]
name=Confluent repository
baseurl=https://packages.confluent.io/rpm/7.6
gpgcheck=1
gpgkey=https://packages.confluent.io/rpm/7.6/archive.key
enabled=1

[Confluent-Clients]
name=Confluent Clients repository
baseurl=https://packages.confluent.io/clients/rpm/centos/9/x86_64
gpgcheck=1
gpgkey=https://packages.confluent.io/clients/rpm/archive.key
enabled=1
13 changes: 13 additions & 0 deletions jpo-conflictvisualizer-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
<groupId>usdot.jpo.ode</groupId>
<artifactId>jpo-conflictvisualizer-api</artifactId>
<version>1.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>jpo-conflictvisualizer-api</name>
<description>Conflict Visualizer</description>
<properties>
Expand Down Expand Up @@ -77,6 +78,12 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.vaadin.external.google</groupId>
<artifactId>android-json</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
Expand Down Expand Up @@ -105,6 +112,11 @@
<artifactId>jpo-ode-plugins</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>usdot.jpo.ode</groupId>
<artifactId>jpo-ode-svcs</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>usdot.jpo.ode</groupId>
<artifactId>jpo-geojsonconverter</artifactId>
Expand Down Expand Up @@ -211,6 +223,7 @@
</dependency>
</dependencies>
<build>
<finalName>${project.artifactId}</finalName>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
package us.dot.its.jpo.ode.api;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.web.servlet.config.annotation.EnableWebMvc;

import us.dot.its.jpo.ode.api.asn1.DecoderManager;
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.scheduling.annotation.EnableScheduling;

@EnableWebMvc
@SpringBootApplication
@EnableScheduling
@ComponentScan(basePackages = {"us.dot.its.jpo.ode.api", "us.dot.its.jpo.geojsonconverter.validator"})
public class ConflictApiApplication extends SpringBootServletInitializer {

@Autowired DecoderManager manager;

@Override
protected SpringApplicationBuilder configure(SpringApplicationBuilder builder) {
return builder.sources(ConflictApiApplication.class);
Expand All @@ -21,9 +28,7 @@ public static void main(String[] args) {
SpringApplication.run(ConflictApiApplication.class, args);
System.out.println("Started Conflict Monitor API");
System.out.println("Conflict Monitor API docs page found here: http://localhost:8081/swagger-ui/index.html");
System.out.println("Startup Complete");


System.out.println("Startup Complete");
}

// @Bean
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@
import us.dot.its.jpo.ode.api.models.IntersectionReferenceData;

public interface ProcessedMapRepository extends DataLoader<ProcessedMap<LineString>>{
Query getQuery(Integer intersectionID, Long startTime, Long endTime,boolean latest, boolean compact);
Query getQuery(Integer intersectionID, Long startTime, Long endTime, boolean latest, boolean compact);

long getQueryResultCount(Query query);

List<ProcessedMap> findProcessedMaps(Query query);
List<ProcessedMap<LineString>> findProcessedMaps(Query query);

List<IntersectionReferenceData> getIntersectionIDs();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.mongodb.MongoException;
import com.mongodb.client.DistinctIterable;
Expand Down Expand Up @@ -50,6 +51,8 @@ public class ProcessedMapRepositoryImpl implements ProcessedMapRepository {
@Autowired
ConflictMonitorApiProperties props;

TypeReference<ProcessedMap<LineString>> processedMapTypeReference = new TypeReference<>(){};

private String collectionName = "ProcessedMap";
private ObjectMapper mapper = DateJsonMapper.getInstance();
private Logger logger = LoggerFactory.getLogger(ProcessedMapRepositoryImpl.class);
Expand Down Expand Up @@ -93,12 +96,12 @@ public long getQueryResultCount(Query query) {
return mongoTemplate.count(query, ProcessedMap.class, collectionName);
}

public List<ProcessedMap> findProcessedMaps(Query query) {
public List<ProcessedMap<LineString>> findProcessedMaps(Query query) {
List<Map> documents = mongoTemplate.find(query, Map.class, collectionName);
List<ProcessedMap> convertedList = new ArrayList<>();
List<ProcessedMap<LineString>> convertedList = new ArrayList<>();
for (Map document : documents) {
document.remove("_id");
ProcessedMap bsm = mapper.convertValue(document, ProcessedMap.class);
ProcessedMap<LineString> bsm = mapper.convertValue(document, processedMapTypeReference);
convertedList.add(bsm);
}
return convertedList;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
package us.dot.its.jpo.ode.api.asn1;

import org.springframework.stereotype.Component;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import us.dot.its.jpo.ode.api.models.messages.BsmDecodedMessage;
import us.dot.its.jpo.ode.api.models.messages.DecodedMessage;
import us.dot.its.jpo.ode.api.models.messages.EncodedMessage;
import us.dot.its.jpo.ode.context.AppContext;
import us.dot.its.jpo.ode.model.Asn1Encoding;
import us.dot.its.jpo.ode.model.Asn1Encoding.EncodingRule;
import us.dot.its.jpo.ode.model.OdeAsn1Data;
import us.dot.its.jpo.ode.model.OdeAsn1Payload;
import us.dot.its.jpo.ode.model.OdeBsmData;
import us.dot.its.jpo.ode.model.OdeBsmMetadata;
import us.dot.its.jpo.ode.model.OdeBsmMetadata.BsmSource;
import us.dot.its.jpo.ode.model.OdeBsmPayload;
import us.dot.its.jpo.ode.model.OdeData;
import us.dot.its.jpo.ode.model.OdeHexByteArray;
import us.dot.its.jpo.ode.model.OdeLogMetadata.RecordType;
import us.dot.its.jpo.ode.model.OdeLogMetadata.SecurityResultCode;
import us.dot.its.jpo.ode.model.OdeLogMsgMetadataLocation;
import us.dot.its.jpo.ode.model.OdeMsgPayload;
import us.dot.its.jpo.ode.model.ReceivedMessageDetails;
import us.dot.its.jpo.ode.model.RxSource;
import us.dot.its.jpo.ode.plugin.j2735.builders.BsmBuilder;
import us.dot.its.jpo.ode.util.JsonUtils;
import us.dot.its.jpo.ode.util.XmlUtils;
import us.dot.its.jpo.ode.util.XmlUtils.XmlUtilsException;


@Component
public class BsmDecoder implements Decoder {


@Override
public DecodedMessage decode(EncodedMessage message) {

// Convert to Ode Data type and Add Metadata
OdeData data = getAsOdeData(message.getAsn1Message());

XmlUtils xmlUtils = new XmlUtils();

try {
// Convert to XML for ASN.1 Decoder
String xml = xmlUtils.toXml(data);

// Send String through ASN.1 Decoder to get Decoded XML Data
String decodedXml = DecoderManager.decodeXmlWithAcm(xml);
// String decodedXml = mockDecodeXmlWithAcm(xml);

// Convert to Ode Json
OdeBsmData bsm = getAsOdeJson(decodedXml);

// build output data structure
DecodedMessage decodedMessage = new BsmDecodedMessage(bsm, message.getAsn1Message(), "");
return decodedMessage;

} catch (JsonProcessingException e) {
e.printStackTrace();
return new BsmDecodedMessage(null, message.getAsn1Message(), e.getMessage());
} catch (Exception e) {
e.printStackTrace();
return new BsmDecodedMessage(null, message.getAsn1Message(), e.getMessage());
}
}

@Override
public OdeData getAsOdeData(String encodedData) {
OdeMsgPayload payload = new OdeAsn1Payload(new OdeHexByteArray(encodedData));

// construct metadata
OdeBsmMetadata metadata = new OdeBsmMetadata(payload);
metadata.setOdeReceivedAt(DecoderManager.getOdeReceivedAt());
metadata.setRecordType(RecordType.bsmTx);
metadata.setSecurityResultCode(SecurityResultCode.success);

// construct metadata: receivedMessageDetails
ReceivedMessageDetails receivedMessageDetails = new ReceivedMessageDetails();
receivedMessageDetails.setRxSource(RxSource.RV);

// construct metadata: locationData
OdeLogMsgMetadataLocation locationData = new OdeLogMsgMetadataLocation();
receivedMessageDetails.setLocationData(locationData);

metadata.setReceivedMessageDetails(receivedMessageDetails);
metadata.setBsmSource(BsmSource.RV);

Asn1Encoding unsecuredDataEncoding = new Asn1Encoding("unsecuredData", "MessageFrame",
EncodingRule.UPER);
metadata.addEncoding(unsecuredDataEncoding);

// construct odeData
return new OdeAsn1Data(metadata, payload);
}

@Override
public OdeBsmData getAsOdeJson(String consumedData) throws XmlUtilsException {
ObjectNode consumed = XmlUtils.toObjectNode(consumedData);

JsonNode metadataNode = consumed.findValue(AppContext.METADATA_STRING);
if (metadataNode instanceof ObjectNode) {
ObjectNode object = (ObjectNode) metadataNode;
object.remove(AppContext.ENCODINGS_STRING);
}

OdeBsmMetadata metadata = (OdeBsmMetadata) JsonUtils.fromJson(
metadataNode.toString(), OdeBsmMetadata.class);

/*
* ODE-755 and ODE-765 Starting with schemaVersion=5 receivedMessageDetails
* will be present in BSM metadata. None should be present in prior versions.
*/
if (metadata.getSchemaVersion() <= 4) {
metadata.setReceivedMessageDetails(null);
}

OdeBsmPayload payload = new OdeBsmPayload(
BsmBuilder.genericBsm(consumed.findValue("BasicSafetyMessage")));
return new OdeBsmData(metadata, payload);
}

public String mockDecodeXmlWithAcm(String xmlMessage){
return "<?xml version=\"1.0\"?><OdeAsn1Data><metadata><bsmSource>RV</bsmSource><logFileName/><recordType>bsmTx</recordType><securityResultCode>success</securityResultCode><receivedMessageDetails><locationData><latitude/><longitude/><elevation/><speed/><heading/></locationData><rxSource>RV</rxSource></receivedMessageDetails><encodings><encodings><elementName>unsecuredData</elementName><elementType>MessageFrame</elementType><encodingRule>UPER</encodingRule></encodings></encodings><payloadType>us.dot.its.jpo.ode.model.OdeAsn1Payload</payloadType><serialId><streamId>8829c539-e684-40b7-a786-692acd3f897a</streamId><bundleSize>1</bundleSize><bundleId>0</bundleId><recordId>0</recordId><serialNumber>0</serialNumber></serialId><odeReceivedAt>2024-05-08T20:47:53.830130272Z</odeReceivedAt><schemaVersion>6</schemaVersion><maxDurationTime>0</maxDurationTime><recordGeneratedAt/><recordGeneratedBy/><sanitized>false</sanitized><odePacketID/><odeTimStartDateTime/><originIp/></metadata><payload><dataType>MessageFrame</dataType><data><MessageFrame><messageId>20</messageId><value><BasicSafetyMessage><coreData><msgCnt>18</msgCnt><id>B42DE51F</id><secMark>2998</secMark><lat>0</lat><long>0</long><elev>0</elev><accuracy><semiMajor>0</semiMajor><semiMinor>0</semiMinor><orientation>8100</orientation></accuracy><transmission><forwardGears/></transmission><speed>1315</speed><heading>6448</heading><angle>-1</angle><accelSet><long>0</long><lat>0</lat><vert>50</vert><yaw>0</yaw></accelSet><brakes><wheelBrakes>00000</wheelBrakes><traction><on/></traction><abs><on/></abs><scs><on/></scs><brakeBoost><unavailable/></brakeBoost><auxBrakes><unavailable/></auxBrakes></brakes><size><width>250</width><length>590</length></size></coreData><partII><BSMpartIIExtension><partII-Id>0</partII-Id><partII-Value><VehicleSafetyExtensions><pathHistory><crumbData><PathHistoryPoint><latOffset>-795</latOffset><lonOffset>2109</lonOffset><elevationOffset>0</elevationOffset><timeOffset>62789</timeOffset></PathHistoryPoint></crumbData></pathHistory><pathPrediction><radiusOfCurve>32767</radiusOfCurve><confidence>180</confidence></pathPrediction></VehicleSafetyExtensions></partII-Value></BSMpartIIExtension><BSMpartIIExtension><partII-Id>2</partII-Id><partII-Value><SupplementalVehicleExtensions><classification>0</classification><classDetails><keyType>0</keyType><hpmsType><none/></hpmsType></classDetails><vehicleData/><weatherReport><isRaining><error/></isRaining><rainRate>65535</rainRate><precipSituation><unknown/></precipSituation><solarRadiation>65535</solarRadiation><friction>101</friction><roadFriction>0</roadFriction></weatherReport><weatherProbe><airTemp>52</airTemp><airPressure>71</airPressure><rainRates><statusFront><off/></statusFront><rateFront>0</rateFront></rainRates></weatherProbe></SupplementalVehicleExtensions></partII-Value></BSMpartIIExtension><BSMpartIIExtension><partII-Id>1</partII-Id><partII-Value><SpecialVehicleExtensions><vehicleAlerts><notUsed>0</notUsed><sirenUse><notInUse/></sirenUse><lightsUse><notInUse/></lightsUse><multi><unavailable/></multi><events><notUsed>0</notUsed><event>1000000000000000</event></events></vehicleAlerts></SpecialVehicleExtensions></partII-Value></BSMpartIIExtension></partII></BasicSafetyMessage></value></MessageFrame></data></payload></OdeAsn1Data>";
}

}
Loading

0 comments on commit b66b67d

Please sign in to comment.