From d6e7c31a882c86f42fe5e7d664562c176115bb11 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Wed, 24 Sep 2014 17:18:47 +0300 Subject: [PATCH 1/6] aidr-analytics REDIS to DB write functionality ready --- .../analysis/api/WriteStatisticsData.java | 233 -------------- .../aidr/analysis/stat/FreqDataMapRecord.java | 17 - .../aidr/analysis/stat/TagDataMapRecord.java | 17 - .../aidr/analysis/utils/MiscUtilities.java | 23 -- .../src/main/resources/granularity.properties | 1 - {aidr-analysis => aidr-analytics}/pom.xml | 0 .../aidr/analysis/api/ApplicationConfig.java | 0 .../analysis/api/GetFrequencyStatistics.java | 0 .../qcri/aidr/analysis/api/GetStatistics.java | 0 .../analysis/api/GetTagDataStatistics.java | 0 .../analysis/api/WriteStatisticsData.java | 296 ++++++++++++++++++ .../aidr/analysis/entity/FrequencyData.java | 0 .../aidr/analysis/entity/FrequencyDataPK.java | 0 .../qa/qcri/aidr/analysis/entity/TagData.java | 18 +- .../qcri/aidr/analysis/entity/TagDataPK.java | 0 .../FrequencyStatisticsResourceFacade.java | 0 .../TagDataStatisticsResourceFacade.java | 0 .../FrequencyStatisticsResourceFacadeImp.java | 8 +- .../TagDataStatisticsResourceFacadeImp.java | 32 +- .../aidr/analysis/stat/FreqDataMapRecord.java | 62 ++++ .../qa/qcri/aidr/analysis/stat/MapRecord.java | 20 ++ .../aidr/analysis/stat/TagDataMapRecord.java | 62 ++++ .../aidr/analysis/utils/CommonOperations.java | 0 .../aidr/analysis/utils/GranularityData.java | 4 +- .../main/resources/META-INF/persistence.xml | 6 +- .../src/main/resources/granularity.properties | 1 + .../src/main/resources/log4j.properties | 0 .../src/main/webapp/META-INF/MANIFEST.MF | 0 .../src/main/webapp/META-INF/context.xml | 0 .../src/main/webapp/WEB-INF/glassfish-web.xml | 0 .../src/main/webapp/WEB-INF/hibernate.cfg.xml | 0 .../src/main/webapp/WEB-INF/web.xml | 0 .../src/main/webapp/index.jsp | 0 .../output/getdata/ChannelBufferManager.java | 22 +- .../qa/qcri/aidr/getdata/InjectorConfig.java | 2 +- 35 files changed, 496 insertions(+), 328 deletions(-) delete mode 100644 aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java delete mode 100644 aidr-analysis/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java delete mode 100644 aidr-analysis/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java delete mode 100644 aidr-analysis/src/main/java/qa/qcri/aidr/analysis/utils/MiscUtilities.java delete mode 100644 aidr-analysis/src/main/resources/granularity.properties rename {aidr-analysis => aidr-analytics}/pom.xml (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/api/GetFrequencyStatistics.java (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/api/GetStatistics.java (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java (100%) create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyData.java (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyDataPK.java (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java (89%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/entity/TagDataPK.java (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/facade/FrequencyStatisticsResourceFacade.java (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java (97%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java (86%) create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/MapRecord.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java (100%) rename {aidr-analysis => aidr-analytics}/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java (92%) rename {aidr-analysis => aidr-analytics}/src/main/resources/META-INF/persistence.xml (90%) create mode 100644 aidr-analytics/src/main/resources/granularity.properties rename {aidr-analysis => aidr-analytics}/src/main/resources/log4j.properties (100%) rename {aidr-analysis => aidr-analytics}/src/main/webapp/META-INF/MANIFEST.MF (100%) rename {aidr-analysis => aidr-analytics}/src/main/webapp/META-INF/context.xml (100%) rename {aidr-analysis => aidr-analytics}/src/main/webapp/WEB-INF/glassfish-web.xml (100%) rename {aidr-analysis => aidr-analytics}/src/main/webapp/WEB-INF/hibernate.cfg.xml (100%) rename {aidr-analysis => aidr-analytics}/src/main/webapp/WEB-INF/web.xml (100%) rename {aidr-analysis => aidr-analytics}/src/main/webapp/index.jsp (100%) diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java deleted file mode 100644 index 609ab350e..000000000 --- a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ /dev/null @@ -1,233 +0,0 @@ -package qa.qcri.aidr.analysis.api; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; - -import javax.ejb.EJB; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.core.Response; - -import org.apache.log4j.Logger; - -import qa.qcri.aidr.analysis.entity.FrequencyData; -import qa.qcri.aidr.analysis.entity.TagData; -import qa.qcri.aidr.analysis.stat.*; -import qa.qcri.aidr.analysis.utils.GranularityData; -import qa.qcri.aidr.analysis.facade.FrequencyStatisticsResourceFacade; -import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; -import qa.qcri.aidr.common.code.DateFormatConfig; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.common.values.ReturnCode; -import qa.qcri.aidr.output.filter.ClassifiedFilteredTweet; -import qa.qcri.aidr.output.filter.NominalLabel; -import qa.qcri.aidr.output.getdata.ChannelBufferManager; - -@Path("/save/") -public class WriteStatisticsData extends ChannelBufferManager implements ServletContextListener { - - // Debugging - private static Logger logger = Logger.getLogger(WriteStatisticsData.class); - private static ErrorLog elog = new ErrorLog(); - - private volatile boolean runFlag = false; - - @EJB - private TagDataStatisticsResourceFacade tagDataEJB; - - @EJB - private FrequencyStatisticsResourceFacade freqDataEJB; - - private static ConcurrentHashMap tagDataMap = null; - private static ConcurrentHashMap freqDataMap = null; - - private Thread t = null; - - @Override - public void contextInitialized(ServletContextEvent sce) { - logger.info("Initializing channel buffer manager"); - System.out.println("[contextInitialized] Initializing channel buffer manager"); - tagDataMap = new ConcurrentHashMap(); - freqDataMap = new ConcurrentHashMap(); - initiateChannelBufferManager(GetStatistics.CHANNEL_REG_EX); - - runFlag = true; - t = new Thread(new WriterThread()); - t.setName("AIDR-Analysis DB Writer Thread"); - t.start(); - - logger.info("Done initializing channel buffer manager"); - System.out.println("[contextInitialized] Done initializing channel buffer manager"); - logger.info("Context Initialized"); - } - - @Override - public void contextDestroyed(ServletContextEvent sce) { - tagDataMap.clear(); - freqDataMap.clear(); - runFlag = false; - super.close(); - if (t.isAlive()) - try { - t.join(); - } catch (InterruptedException e) { - System.out.println("Thread join interrupted for thread: " + t.getName()); - e.printStackTrace(); - } - - logger.info("Context destroyed"); - } - - @POST - @Path("/tagData") - public Map writeDataToTagDataDB(TagData tagData) { - Map response = new HashMap(); - ReturnCode retVal = tagDataEJB.writeData(tagData); - response.put("code", retVal); - return response; - } - - @POST - @Path("/freqData") - public Map writeDataToFreqDataDB(FrequencyData freqData) { - Map response = new HashMap(); - ReturnCode retVal = freqDataEJB.writeData(freqData); - response.put("code", retVal); - return response; - } - - private ReturnCode writeOutputDataToTagDataDB(Long granularity, Long timestamp) { - TreeMap tempMap = new TreeMap(); - try { - tempMap.putAll(tagDataMap); - for (String key: tempMap.keySet()) { - String[] data = key.split("_"); - // data[0] = channelName - // data[1] = attributeCode - // data[2] = labelCode - TagData t = new TagData(data[0], timestamp, granularity, data[1], data[2], ((TagDataMapRecord) tempMap.get(key)).getCount()); - tagDataEJB.writeData(t); - } - } catch (Exception e) { - System.err.println("[writeOutputDataToTagDataDB] Error in writing to TagDataDB table!"); - return ReturnCode.FAIL; - } - return ReturnCode.SUCCESS; - } - - private ReturnCode writeOutputDataToFreqDataDB(Long granularity, Long timestamp) { - TreeMap tempMap = new TreeMap(); - try { - tempMap.putAll(tagDataMap); - for (String key: tempMap.keySet()) { - String[] data = key.split("_"); - // data[0] = channelName - // data[1] = attributeCode - // data[2] = labelCode - // data[3] = bin - FrequencyData t = new FrequencyData(data[0], timestamp, granularity, data[1], data[2], Integer.parseInt(data[3]), ((FreqDataMapRecord) tempMap.get(key)).getCount()); - freqDataEJB.writeData(t); - } - } catch (Exception e) { - System.err.println("[writeOutputDataToFreqDataDB] Error in writing to FrequencyDataDB table!"); - return ReturnCode.FAIL; - } - return ReturnCode.SUCCESS; - } - - private String getBinNumber(float confidence) { - int bin = 0; - for (int i = 10;i <= 100;i += 10) { - if (i > (confidence * 100)) { - return Integer.toString(bin); - } - ++bin; - } - return Integer.toString(bin-1); - } - - /** - * This method is the 'producer' - producing statistics data to be written to the respective tables - * - * @param subscriptionPattern REDIS pattern that Jedis is subscribed to - * @param channelName Name of the channel on which message received from REDIS - * @param receivedMessage Received message for the given channelName - */ - @Override - public void manageChannelBuffers(final String subscriptionPattern, final String channelName, final String receivedMessage) { - System.out.println("Firing manageChannelBuffer on message from channel: " + channelName); - if (null == channelName) { - logger.error("Something terribly wrong! Fatal error in: " + channelName); - } else { - ClassifiedFilteredTweet classifiedTweet = new ClassifiedFilteredTweet().deserialize(receivedMessage); - if (classifiedTweet.getNominalLabels() != null) { - for (NominalLabel nb: classifiedTweet.getNominalLabels()) { - String keyPrefix = channelName + "_" + nb.attribute_code + "_" + nb.label_code; - String tagDataKey = keyPrefix; - String freqDataKey = keyPrefix + "_" + getBinNumber(nb.confidence); - if (tagDataMap.contains(tagDataKey)) { - tagDataMap.put(tagDataKey, ((TagDataMapRecord) tagDataMap.get(tagDataKey)).getCount()+1); - } else { - TagDataMapRecord t = new TagDataMapRecord(); - tagDataMap.put(tagDataKey, t); - } - if (freqDataMap.contains(freqDataKey)) { - freqDataMap.put(freqDataKey, ((FreqDataMapRecord) freqDataMap.get(freqDataKey)).getCount()+1); - } else { - FreqDataMapRecord t = new FreqDataMapRecord(); - freqDataMap.put(freqDataKey, t); - } - } - } - } - } - - private class WriterThread implements Runnable { - - /** - * The 'consumer' method - consumes the statistics data produced by writing to respective tables in DB - */ - @Override - public void run() { - System.out.println("Started writer thread: " + t.getName()); - List granularityList = GranularityData.getGranularities(); - Map lastTagDataWriteTime = new TreeMap(); - Map lastFreqDataWriteTime = new TreeMap(); - for (Long g: granularityList) { - lastTagDataWriteTime.put(g, 0L); - lastFreqDataWriteTime.put(g, 0L); - } - while (runFlag) { - long currentTime = System.currentTimeMillis(); - if (granularityList != null) { - for (Long granularity: granularityList) { - if (0 == lastTagDataWriteTime.get(granularity) || (currentTime - lastTagDataWriteTime.get(granularity)) >= granularity) { - // Write to DB table - ReturnCode retVal = writeOutputDataToTagDataDB(granularity, currentTime); - lastTagDataWriteTime.put(granularity, currentTime); - } - if (0 == lastFreqDataWriteTime.get(granularity) || (currentTime - lastFreqDataWriteTime.get(granularity)) >= granularity) { - // Write to DB table - ReturnCode retVal = writeOutputDataToFreqDataDB(granularity, currentTime); - lastFreqDataWriteTime.put(granularity, currentTime); - } - } - tagDataMap.clear(); - freqDataMap.clear(); - try { - Thread.sleep(granularityList.get(0)-100); - } catch (InterruptedException e) {} - } - } - System.out.println("Done work - exiting thread: " + t.getName()); - } - } - -} diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java b/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java deleted file mode 100644 index 4e32ee80e..000000000 --- a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java +++ /dev/null @@ -1,17 +0,0 @@ -package qa.qcri.aidr.analysis.stat; - -public class FreqDataMapRecord { - private int count; - - public FreqDataMapRecord() { - this.count = 0; - } - - public int getCount() { - return count; - } - - public void setCount( int count) { - this.count = count; - } -} diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java b/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java deleted file mode 100644 index 7ea81cedc..000000000 --- a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java +++ /dev/null @@ -1,17 +0,0 @@ -package qa.qcri.aidr.analysis.stat; - -public class TagDataMapRecord { - private int count; - - public TagDataMapRecord() { - this.count = 0; - } - - public int getCount() { - return count; - } - - public void setCount( int count) { - this.count = count; - } -} diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/utils/MiscUtilities.java b/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/utils/MiscUtilities.java deleted file mode 100644 index f5672f3a9..000000000 --- a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/utils/MiscUtilities.java +++ /dev/null @@ -1,23 +0,0 @@ -package qa.qcri.aidr.analysis.utils; - -public class MiscUtilities { - - public static long parseTime(String timeString) { - long duration = 0; - - float value = Float.parseFloat(timeString.substring(0, timeString.length()-1)); - if (value > 0) { - String suffix = timeString.substring(timeString.length() - 1, timeString.length()); - if (suffix.equalsIgnoreCase("s")) - duration = Math.round(value * 1000); - if (suffix.equalsIgnoreCase("m")) - duration = Math.round(value * 1000 * 60); - if (suffix.equalsIgnoreCase("h")) - duration = Math.round(value * 1000 * 60 * 60); - if (suffix.equalsIgnoreCase("d")) - duration = Math.round(value * 1000 * 60 * 60 * 24); - } - return duration; - } - -} diff --git a/aidr-analysis/src/main/resources/granularity.properties b/aidr-analysis/src/main/resources/granularity.properties deleted file mode 100644 index 8fe24e3d3..000000000 --- a/aidr-analysis/src/main/resources/granularity.properties +++ /dev/null @@ -1 +0,0 @@ -granularity=5m,1h,1d \ No newline at end of file diff --git a/aidr-analysis/pom.xml b/aidr-analytics/pom.xml similarity index 100% rename from aidr-analysis/pom.xml rename to aidr-analytics/pom.xml diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/GetFrequencyStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetFrequencyStatistics.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/GetFrequencyStatistics.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetFrequencyStatistics.java diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/GetStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetStatistics.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/GetStatistics.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetStatistics.java diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java new file mode 100644 index 000000000..23c9fd232 --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -0,0 +1,296 @@ +package qa.qcri.aidr.analysis.api; + + +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; + +import javax.ejb.EJB; +import javax.json.Json; +import javax.json.JsonObject; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.log4j.Logger; + +import qa.qcri.aidr.analysis.entity.FrequencyData; +import qa.qcri.aidr.analysis.entity.TagData; +import qa.qcri.aidr.analysis.stat.*; +import qa.qcri.aidr.analysis.utils.GranularityData; +import qa.qcri.aidr.analysis.facade.FrequencyStatisticsResourceFacade; +import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; +import qa.qcri.aidr.common.logging.ErrorLog; +import qa.qcri.aidr.common.values.ReturnCode; +import qa.qcri.aidr.output.filter.ClassifiedFilteredTweet; +import qa.qcri.aidr.output.filter.NominalLabel; +import qa.qcri.aidr.output.getdata.ChannelBufferManager; + +@Path("/save/") +public class WriteStatisticsData extends ChannelBufferManager implements ServletContextListener { + + // Debugging + private static Logger logger = Logger.getLogger(WriteStatisticsData.class); + private static ErrorLog elog = new ErrorLog(); + + private volatile boolean runFlag = false; + + @EJB + private TagDataStatisticsResourceFacade tagDataEJB; + + @EJB + private FrequencyStatisticsResourceFacade freqDataEJB; + + private ExecutorService executorServicePool = null; + private Thread t = null; + + private static final String SENTINEL = "#"; + private static ConcurrentHashMap tagDataMap = null; + private static ConcurrentHashMap freqDataMap = null; + private static ConcurrentHashMap channelMap = null; + private List granularityList = null; + private long lastTagDataCheckedTime = 0; + private long lastFreqDataCheckedTime = 0; + + @Override + public void contextInitialized(ServletContextEvent sce) { + logger.info("Initializing channel buffer manager"); + System.out.println("[contextInitialized] Initializing channel buffer manager"); + tagDataMap = new ConcurrentHashMap(); + freqDataMap = new ConcurrentHashMap(); + channelMap = new ConcurrentHashMap(); + + granularityList = GranularityData.getGranularities(); + + initiateChannelBufferManager(GetStatistics.CHANNEL_REG_EX); + + runFlag = true; + t = new Thread(new WriterThread()); + t.setName("AIDR-Analysis DB Writer Thread"); + executorServicePool = getExecutorServicePool(); + if (executorServicePool != null) { + executorServicePool.submit(t); + } + logger.info("Done initializing channel buffer manager"); + System.out.println("[contextInitialized] Done initializing channel buffer manager"); + logger.info("Context Initialized"); + } + + @Override + public void contextDestroyed(ServletContextEvent sce) { + tagDataMap.clear(); + freqDataMap.clear(); + runFlag = false; + close(); + logger.info("Context destroyed"); + } + + @GET + @Path("/ping") + @Produces(MediaType.APPLICATION_JSON) + public Response ping() { + JsonObject obj = Json.createObjectBuilder() + .add("aidr-analysis/writeData", "RUNNING") + .build(); + + return Response.ok(obj).build(); + } + + private ReturnCode writeOutputDataToTagDataDB(Long granularity, Long timestamp) { + System.out.println("tagDataMap size: " + (tagDataMap != null ? tagDataMap.size() : "null")); + try { + for (String key: tagDataMap.keySet()) { + String[] data = key.split(SENTINEL); + // data[0] = channelName + // data[1] = attributeCode + // data[2] = labelCode + TagDataMapRecord tCount = (TagDataMapRecord) tagDataMap.get(key); + TagData t = new TagData(data[0], timestamp, granularity, data[1], data[2], tCount.getCount(granularity)); + System.out.println("Will attempt to persist data for: " + t.getCrisisCode() + ", " + t.getAttributeCode() + + ", " + t.getLabelCode() + ", " + t.getTimestamp() + ", " + t.getGranularity() + ": " + t.getCount()); + tagDataEJB.writeData(t); + tCount.resetCount(granularity); + TagDataMapRecord temp = (TagDataMapRecord) tagDataMap.get(key); + System.out.println("[writeOutputDataToTagDataDB] After reset, count for key = " + key + " is = " + temp.getCount(granularity)); + + } + } catch (Exception e) { + System.err.println("[writeOutputDataToTagDataDB] Error in writing to TagDataDB table!"); + e.printStackTrace(); + return ReturnCode.FAIL; + } + return ReturnCode.SUCCESS; + } + + private ReturnCode writeOutputDataToFreqDataDB(Long granularity, Long timestamp) { + try { + for (String key: freqDataMap.keySet()) { + String[] data = key.split(SENTINEL); + // data[0] = channelName + // data[1] = attributeCode + // data[2] = labelCode + // data[3] = bin + FreqDataMapRecord fCount = (FreqDataMapRecord) freqDataMap.get(key); + FrequencyData f = new FrequencyData(data[0], timestamp, granularity, data[1], data[2], Integer.parseInt(data[3]), fCount.getCount(granularity)); + freqDataEJB.writeData(f); + fCount.resetCount(granularity); + FreqDataMapRecord temp = (FreqDataMapRecord) freqDataMap.get(key); + System.out.println("[writeOutputDataToFreqDataDB] After reset, count for key = " + key + " is = " + temp.getCount(granularity)); + } + } catch (Exception e) { + System.err.println("[writeOutputDataToFreqDataDB] Error in writing to FrequencyDataDB table!"); + e.printStackTrace(); + return ReturnCode.FAIL; + } + return ReturnCode.SUCCESS; + } + + private String getBinNumber(float confidence) { + int bin = 0; + for (int i = 10;i <= 100;i += 10) { + if (i > (confidence * 100)) { + return Integer.toString(bin); + } + ++bin; + } + return Integer.toString(bin-1); + } + + /** + * This method is the 'producer' - producing statistics data to be written to the respective tables + * + * @param subscriptionPattern REDIS pattern that Jedis is subscribed to + * @param channelName Name of the channel on which message received from REDIS + * @param receivedMessage Received message for the given channelName + */ + @Override + public void manageChannelBuffersWrapper(final String subscriptionPattern, final String channelName, final String receivedMessage) { + //System.out.println("Firing manageChannelBuffer on message from channel: " + channelName); + manageChannelBuffers(subscriptionPattern, channelName, receivedMessage); + if (null == channelName) { + logger.error("Something terribly wrong! Fatal error in: " + channelName); + } else { + try { + ClassifiedFilteredTweet classifiedTweet = new ClassifiedFilteredTweet().deserialize(receivedMessage); + if (classifiedTweet != null && classifiedTweet.getNominalLabels() != null) { + channelMap.putIfAbsent(classifiedTweet.getCrisisCode(), System.currentTimeMillis()); + + for (NominalLabel nb: classifiedTweet.getNominalLabels()) { + StringBuffer keyPrefix = new StringBuffer() + .append(classifiedTweet.getCrisisCode()).append(SENTINEL) + .append(nb.attribute_code).append(SENTINEL) + .append(nb.label_code); + String tagDataKey = new String(keyPrefix); + String freqDataKey = new String(keyPrefix.append(SENTINEL).append(getBinNumber(nb.confidence))); + if (tagDataMap.containsKey(tagDataKey)) { + TagDataMapRecord t = (TagDataMapRecord) tagDataMap.get(tagDataKey); + t.incrementAllCounts(); + tagDataMap.put(tagDataKey, t); + } else { + TagDataMapRecord t = new TagDataMapRecord(granularityList); + tagDataMap.put(tagDataKey, t); + System.out.println("[manageChannelBuffersWrapper] New Tag map entry with key: " + tagDataKey + " value = " + tagDataMap.get(tagDataKey)); + } + if (freqDataMap.containsKey(freqDataKey)) { + FreqDataMapRecord f = (FreqDataMapRecord) freqDataMap.get(freqDataKey); + f.incrementAllCounts(); + freqDataMap.put(freqDataKey, f); + } else { + FreqDataMapRecord t = new FreqDataMapRecord(granularityList); + freqDataMap.put(freqDataKey, t); + System.out.println("[manageChannelBuffersWrapper] New Freq map entry with key: " + freqDataKey + " value = " + freqDataMap.get(freqDataKey)); + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + // Periodically check if any channel is down - if so, delete all in-memory data for that channel + lastTagDataCheckedTime = periodicInactiveChannelCheck(lastTagDataCheckedTime, tagDataMap); + lastFreqDataCheckedTime = periodicInactiveChannelCheck(lastFreqDataCheckedTime, freqDataMap); + } + + private long periodicInactiveChannelCheck(long lastCheckedTime, ConcurrentHashMap dataMap) { + long currentTime = System.currentTimeMillis(); + if (currentTime - lastCheckedTime > CHECK_INTERVAL) { + for (String key: channelMap.keySet()) { + if ((currentTime - channelMap.get(key)) > NO_DATA_TIMEOUT) { + System.out.println("[periodicInactiveChannelCheck] Deleting data for inactive channel = " + key); + int deleteCount = deleteMapRecords(key, dataMap); + System.out.println("[periodicInactiveChannelCheck] Deleted records count for inactive channel <" + key + "> is = " + deleteCount); + } + } + } + return currentTime; + } + + private int deleteMapRecords(final String key, ConcurrentHashMap dataMap) { + int count = 0; + for (String keyVal: dataMap.keySet()) { + MapRecord data = (MapRecord) dataMap.get(key); + synchronized(dataMap) { + if (keyVal.contains(key) && data != null && data.isCountZeroForAllGranularity()) { + dataMap.remove(keyVal); + ++count; + } + } + } + return count; + } + + + private class WriterThread implements Runnable { + + /** + * The 'consumer' method - consumes the statistics data produced by writing to respective tables in DB + */ + @Override + public void run() { + System.out.println("Started writer thread: " + t.getName()); + Map lastTagDataWriteTime = new TreeMap(); + Map lastFreqDataWriteTime = new TreeMap(); + for (Long g: granularityList) { + lastTagDataWriteTime.put(g, 0L); + lastFreqDataWriteTime.put(g, 0L); + } + while (runFlag) { + long currentTime = System.currentTimeMillis(); + if (granularityList != null) { + for (Long granularity: granularityList) { + if (0 == lastTagDataWriteTime.get(granularity) || (currentTime - lastTagDataWriteTime.get(granularity)) >= granularity) { + // Write to DB table + ReturnCode retVal = writeOutputDataToTagDataDB(granularity, currentTime); + lastTagDataWriteTime.put(granularity, currentTime); + System.out.println("[ToTagDataDB] retVal = " + retVal); + if (ReturnCode.SUCCESS.equals(retVal)) { + System.out.println("[ToTagDataDB] Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); + } + } + if (0 == lastFreqDataWriteTime.get(granularity) || (currentTime - lastFreqDataWriteTime.get(granularity)) >= granularity) { + // Write to DB table + ReturnCode retVal = writeOutputDataToFreqDataDB(granularity, currentTime); + System.out.println("[ToFreqDataDB] retVal = " + retVal); + lastFreqDataWriteTime.put(granularity, currentTime); + if (ReturnCode.SUCCESS.equals(retVal)) { + System.out.println("[ToFreqDataDB] Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); + } + } + } + try { + Thread.sleep(granularityList.get(0)-1000); // sleep for the minimum granularity period + } catch (InterruptedException e) {} + } + } + System.out.println("Done work - exiting thread: " + t.getName()); + } + } + +} diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyData.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyData.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyData.java diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyDataPK.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyDataPK.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyDataPK.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyDataPK.java diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java similarity index 89% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java index 5dc239583..ad593a761 100644 --- a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java @@ -56,12 +56,12 @@ public class TagData implements Serializable { private Integer count; @XmlElement - @Column(name="min_create_at", nullable=true) - private Long minCreateAt; + @Column(name="min_created_at", nullable=true) + private Long minCreatedAt; @XmlElement - @Column(name="max_create_at", nullable=true) - private Long maxCreateAt; + @Column(name="max_created_at", nullable=true) + private Long maxCreatedAt; public TagData() {} @@ -131,19 +131,19 @@ public void setCount(Integer count) { this.count = count; } - public Long getMinCreateAt() { - return this.minCreateAt; + public Long getMinCreatedAt() { + return this.minCreatedAt; } public void setMinCreatedAt(Long minCreatedAt) { - this.minCreateAt = minCreatedAt; + this.minCreatedAt = minCreatedAt; } public Long getMaxCreatedAt() { - return this.maxCreateAt; + return this.maxCreatedAt; } public void setMaxCreatedAt(Long maxCreatedAt) { - this.maxCreateAt = maxCreatedAt; + this.maxCreatedAt = maxCreatedAt; } } diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/entity/TagDataPK.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/TagDataPK.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/entity/TagDataPK.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/TagDataPK.java diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/FrequencyStatisticsResourceFacade.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/FrequencyStatisticsResourceFacade.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/FrequencyStatisticsResourceFacade.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/FrequencyStatisticsResourceFacade.java diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java similarity index 97% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java index d998a5a95..917fcd2b7 100644 --- a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java @@ -33,9 +33,15 @@ public class FrequencyStatisticsResourceFacadeImp extends CommonOperations imple public ReturnCode writeData(FrequencyData freqData) { try { em.persist(freqData); + //System.out.println("Success in persisting data for: " + freqData.getCrisisCode() + ", " + freqData.getAttributeCode() + // + ", " + freqData.getLabelCode() + ", " + freqData.getTimestamp() + ", " + freqData.getGranularity() + // + ", " + freqData.getBin() + ": " + freqData.getCount()); return ReturnCode.SUCCESS; } catch (Exception e) { - logger.info("Error in persisting data for: " + freqData.getCrisisCode() + ", " + freqData.getAttributeCode() + ", " + freqData.getLabelCode()); + System.out.println("Failure in persisting data for: " + freqData.getCrisisCode() + ", " + freqData.getAttributeCode() + + ", " + freqData.getLabelCode() + ", " + freqData.getTimestamp() + ", " + freqData.getGranularity() + + ", " + freqData.getBin() + ": " + freqData.getCount()); + e.printStackTrace(); logger.error(elog.toStringException(e)); return ReturnCode.FAIL; } diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java similarity index 86% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java index 3d0a61a25..ae139dd0f 100644 --- a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java @@ -23,8 +23,8 @@ @Stateless public class TagDataStatisticsResourceFacadeImp extends CommonOperations implements TagDataStatisticsResourceFacade { - private static Logger logger = LoggerFactory.getLogger(TagDataStatisticsResourceFacadeImp.class); - private static ErrorLog elog = new ErrorLog(); + //private static Logger logger = LoggerFactory.getLogger(TagDataStatisticsResourceFacadeImp.class); + //private static ErrorLog elog = new ErrorLog(); @PersistenceContext(unitName = "qa.qcri.aidr.analysis-EJBS") private EntityManager em; @@ -32,10 +32,14 @@ public class TagDataStatisticsResourceFacadeImp extends CommonOperations impleme public ReturnCode writeData(TagData tagData) { try { em.persist(tagData); + //System.out.println("Success in persisting data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + // + ", " + tagData.getLabelCode() + ", " + tagData.getTimestamp() + ", " + tagData.getGranularity() + ": " + tagData.getCount()); return ReturnCode.SUCCESS; } catch (Exception e) { - logger.info("Error in persisting data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + ", " + tagData.getLabelCode()); - logger.error(elog.toStringException(e)); + System.err.println("Failure in persisting data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + + ", " + tagData.getLabelCode() + ", " + tagData.getTimestamp() + ", " + tagData.getGranularity() + ": " + tagData.getCount()); + e.printStackTrace(); + //logger.error(elog.toStringException(e)); return ReturnCode.FAIL; } } @@ -54,7 +58,7 @@ public TagData getSingleDataByPK(TagDataPK tagDataPK) { TagData obj = (TagData) criteria.uniqueResult(); return obj; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } @@ -67,7 +71,7 @@ public List getDataByCrisis(String crisisCode) { List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } @@ -84,7 +88,7 @@ public List getDataByCrisisAttributeLabel(String crisisCode, String att List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } @@ -102,7 +106,7 @@ public List getDataByCrisisAttributeLabelGranularity(String crisisCode, List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } @@ -121,7 +125,7 @@ public List getDataAfterTimestamp(String crisisCode, String attributeCo List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } @@ -142,7 +146,7 @@ public List getDataAfterTimestampGranularity(String crisisCode, String List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } @@ -161,7 +165,7 @@ public List getDataBeforeTimestamp(String crisisCode, String attributeC List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } @@ -182,7 +186,7 @@ public List getDataBeforeTimestampGranularity(String crisisCode, String List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } @@ -203,7 +207,7 @@ public List getDataInInterval(String crisisCode, String attributeCode, List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } @@ -225,7 +229,7 @@ public List getDataInIntervalWithGranularity(String crisisCode, String List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + //logger.error(elog.toStringException(e)); } return null; } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java new file mode 100644 index 000000000..93ef8072b --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java @@ -0,0 +1,62 @@ +package qa.qcri.aidr.analysis.stat; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; + +public class FreqDataMapRecord extends MapRecord { + + public FreqDataMapRecord(List granularityList) { + this.count = new ConcurrentHashMap(); + for (Long g: granularityList) { + this.count.put(g, 0); + } + this.lastUpdateTime = System.currentTimeMillis(); + } + + public int getCount(Long g) { + return count.containsKey(g) ? count.get(g) : 0; + } + + public void setCount(Long g, int val) { + synchronized(count) { + count.put(g, val); + } + this.lastUpdateTime = System.currentTimeMillis(); + } + + public void incrementCount(Long g) { + this.setCount(g, count.get(g)+1); + } + + public void incrementAllCounts() { + for (Long g: count.keySet()) { + this.setCount(g, count.get(g)+1); + } + } + + public void resetCount(Long g) { + this.setCount(g, 0); + } + + public long getLastUpdateTime() { + return this.lastUpdateTime; + } + + public void setLastUpdateTime(long lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public boolean isCountZeroForGranularity(Long g) { + return ((count.get(g) == 0) || (count.get(g) == null)); + } + + public boolean isCountZeroForAllGranularity() { + boolean flag = true; + for (Long g: count.keySet()) { + flag &= isCountZeroForGranularity(g); + } + return flag; + } +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/MapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/MapRecord.java new file mode 100644 index 000000000..187ad4191 --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/MapRecord.java @@ -0,0 +1,20 @@ +package qa.qcri.aidr.analysis.stat; + +import java.util.Map; + +public abstract class MapRecord { + public Map count = null; + public long lastUpdateTime = 0; + + public abstract int getCount(Long g); + public abstract void setCount(Long g, int val); + public abstract void incrementCount(Long g); + public abstract void incrementAllCounts(); + public abstract void resetCount(Long g); + public abstract boolean isCountZeroForGranularity(Long g); + public abstract boolean isCountZeroForAllGranularity(); + + public abstract long getLastUpdateTime(); + public abstract void setLastUpdateTime(long lastUpdateTime); + +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java new file mode 100644 index 000000000..aa86baf9f --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java @@ -0,0 +1,62 @@ +package qa.qcri.aidr.analysis.stat; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + + +public class TagDataMapRecord extends MapRecord { + + public TagDataMapRecord(List granularityList) { + this.count = new ConcurrentHashMap(); + for (Long g: granularityList) { + this.count.put(g, 0); + } + this.lastUpdateTime = System.currentTimeMillis(); + } + + public int getCount(Long g) { + return count.containsKey(g) ? count.get(g) : 0; + } + + public void setCount(Long g, int val) { + synchronized(count) { + count.put(g, val); + } + this.lastUpdateTime = System.currentTimeMillis(); + } + + public void incrementCount(Long g) { + this.setCount(g, count.get(g)+1); + } + + public void incrementAllCounts() { + for (Long g: count.keySet()) { + this.setCount(g, count.get(g)+1); + } + } + + public void resetCount(Long g) { + this.setCount(g, 0); + } + + public long getLastUpdateTime() { + return this.lastUpdateTime; + } + + public void setLastUpdateTime(long lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public boolean isCountZeroForGranularity(Long g) { + return ((count.get(g) == 0) || (count.get(g) == null)); + } + + public boolean isCountZeroForAllGranularity() { + boolean flag = true; + for (Long g: count.keySet()) { + flag &= isCountZeroForGranularity(g); + } + return flag; + } +} diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java similarity index 100% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java diff --git a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java similarity index 92% rename from aidr-analysis/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java index 6359adea9..b92b7f889 100644 --- a/aidr-analysis/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java @@ -20,7 +20,7 @@ public class GranularityData { private static Logger logger = Logger.getLogger(GranularityData.class); private static ErrorLog elog = new ErrorLog(); - private static String configloadFileName = "granularities.properties"; + private static String configloadFileName = "granularity.properties"; public static List getGranularities() { List granularityList = new ArrayList(); @@ -43,7 +43,7 @@ public static HashMap getConfigProperties(final String fileName) HashMapproperties = new HashMap(); try { //input = new FileInputStream(fileName); - input = GranularityData.class.getClass().getClassLoader().getResourceAsStream(fileName); + input = GranularityData.class.getClassLoader().getResourceAsStream(fileName); // load a properties file prop.load(input); diff --git a/aidr-analysis/src/main/resources/META-INF/persistence.xml b/aidr-analytics/src/main/resources/META-INF/persistence.xml similarity index 90% rename from aidr-analysis/src/main/resources/META-INF/persistence.xml rename to aidr-analytics/src/main/resources/META-INF/persistence.xml index 43d65d093..a61208031 100644 --- a/aidr-analysis/src/main/resources/META-INF/persistence.xml +++ b/aidr-analytics/src/main/resources/META-INF/persistence.xml @@ -9,7 +9,7 @@ - + @@ -33,8 +33,8 @@ - - + + diff --git a/aidr-analytics/src/main/resources/granularity.properties b/aidr-analytics/src/main/resources/granularity.properties new file mode 100644 index 000000000..f3231103c --- /dev/null +++ b/aidr-analytics/src/main/resources/granularity.properties @@ -0,0 +1 @@ +granularity=30s,2m,1h \ No newline at end of file diff --git a/aidr-analysis/src/main/resources/log4j.properties b/aidr-analytics/src/main/resources/log4j.properties similarity index 100% rename from aidr-analysis/src/main/resources/log4j.properties rename to aidr-analytics/src/main/resources/log4j.properties diff --git a/aidr-analysis/src/main/webapp/META-INF/MANIFEST.MF b/aidr-analytics/src/main/webapp/META-INF/MANIFEST.MF similarity index 100% rename from aidr-analysis/src/main/webapp/META-INF/MANIFEST.MF rename to aidr-analytics/src/main/webapp/META-INF/MANIFEST.MF diff --git a/aidr-analysis/src/main/webapp/META-INF/context.xml b/aidr-analytics/src/main/webapp/META-INF/context.xml similarity index 100% rename from aidr-analysis/src/main/webapp/META-INF/context.xml rename to aidr-analytics/src/main/webapp/META-INF/context.xml diff --git a/aidr-analysis/src/main/webapp/WEB-INF/glassfish-web.xml b/aidr-analytics/src/main/webapp/WEB-INF/glassfish-web.xml similarity index 100% rename from aidr-analysis/src/main/webapp/WEB-INF/glassfish-web.xml rename to aidr-analytics/src/main/webapp/WEB-INF/glassfish-web.xml diff --git a/aidr-analysis/src/main/webapp/WEB-INF/hibernate.cfg.xml b/aidr-analytics/src/main/webapp/WEB-INF/hibernate.cfg.xml similarity index 100% rename from aidr-analysis/src/main/webapp/WEB-INF/hibernate.cfg.xml rename to aidr-analytics/src/main/webapp/WEB-INF/hibernate.cfg.xml diff --git a/aidr-analysis/src/main/webapp/WEB-INF/web.xml b/aidr-analytics/src/main/webapp/WEB-INF/web.xml similarity index 100% rename from aidr-analysis/src/main/webapp/WEB-INF/web.xml rename to aidr-analytics/src/main/webapp/WEB-INF/web.xml diff --git a/aidr-analysis/src/main/webapp/index.jsp b/aidr-analytics/src/main/webapp/index.jsp similarity index 100% rename from aidr-analysis/src/main/webapp/index.jsp rename to aidr-analytics/src/main/webapp/index.jsp diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java index 01a1222d4..d023eefd9 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java @@ -48,9 +48,9 @@ public class ChannelBufferManager { - private static final int NO_DATA_TIMEOUT = 48 * 60 * 60 * 1000; // when to delete a channel buffer - private static final int CHECK_INTERVAL = NO_DATA_TIMEOUT; - private static final int CHECK_CHANNEL_PUBLIC_INTERVAL = 5 * 60 * 1000; + public static final int NO_DATA_TIMEOUT = 48 * 60 * 60 * 1000; // when to delete a channel buffer + public static final int CHECK_INTERVAL = NO_DATA_TIMEOUT; + public static final int CHECK_CHANNEL_PUBLIC_INTERVAL = 5 * 60 * 1000; private static int PERSISTER_LOAD_LIMIT; private static int PERSISTER_LOAD_CHECK_INTERVAL_MINUTES; @@ -79,7 +79,7 @@ public class ChannelBufferManager { private static int bufferSize = -1; // Channel Buffering Algorithm related - private static final String CHANNEL_PREFIX_STRING = "aidr_predict."; + public static final String CHANNEL_PREFIX_STRING = "aidr_predict."; public static ConcurrentHashMap subscribedChannels; // DB access related @@ -147,19 +147,27 @@ public void initiateChannelBufferManager(final String channelRegEx) { } } - + + public ExecutorService getExecutorServicePool() { + return executorServicePool; + } + public void initiateChannelBufferManager(final int bufferSize, final String channelRegEx) { initiateChannelBufferManager(channelRegEx); // call default constructor this.bufferSize = bufferSize; // set buffer size } + public void manageChannelBuffersWrapper(final String subscriptionPattern, final String channelName, + final String receivedMessage) { + manageChannelBuffers(subscriptionPattern, channelName, receivedMessage); + } // Does all the essential work: // 1. Searches received message to see if channel name present. // 2. If channel present then simply adds receivedMessage to that channel. // 3. Else, first calls createChannelBuffer() and then executes step (2). // 4. Deletes channelName and channel buffer if channelName not seen for TIMEOUT duration. - private void manageChannelBuffers(final String subscriptionPattern, final String channelName, + public void manageChannelBuffers(final String subscriptionPattern, final String channelName, final String receivedMessage) { if (null == channelName) { logger.error("Something terribly wrong! Fatal error in: " + channelName); @@ -620,7 +628,7 @@ public void onPMessage(String pattern, String channel, String message) { logger.info("Created new redis load shedder for channel: " + channel); } if (redisLoadShedder.get(channel).canProcess(channel)) { - manageChannelBuffers(pattern, channel, message); + manageChannelBuffersWrapper(pattern, channel, message); } } catch (Exception e) { logger.error("Exception occurred, redisLoadShedder = " + redisLoadShedder + ", channel status: " + redisLoadShedder.containsKey(channel)); diff --git a/aidr-output/src/test/java/qa/qcri/aidr/getdata/InjectorConfig.java b/aidr-output/src/test/java/qa/qcri/aidr/getdata/InjectorConfig.java index 327ffa590..40b7fede3 100644 --- a/aidr-output/src/test/java/qa/qcri/aidr/getdata/InjectorConfig.java +++ b/aidr-output/src/test/java/qa/qcri/aidr/getdata/InjectorConfig.java @@ -10,7 +10,7 @@ public class InjectorConfig { public final String channelPrefix = "aidr_predict."; public final String collectionCode = "mock_collection"; //2014-07-palestine_conflict"; - public final int tweets_per_sec = 100; // -1 = no throttle control + public final int tweets_per_sec = 2; // -1 = no throttle control public final int duration = 20; // in mins public final int threads = 5; From 57f50b390abf087af9cdf4a0028c69b97f8d6010 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Thu, 25 Sep 2014 11:14:25 +0300 Subject: [PATCH 2/6] aidr-analytics: REST APIs WIP --- .../analysis/api/GetTagDataStatistics.java | 108 +++++++++++++++--- .../analysis/api/WriteStatisticsData.java | 2 +- .../qcri/aidr/analysis/dto/TagCountDTO.java | 40 +++++++ .../aidr/analysis/dto/TagCountSeriesDTO.java | 51 +++++++++ .../analysis/dto/TimeWindowTagCountDTO.java | 40 +++++++ .../dto/helper/TagCountDTOHelper.java | 22 ++++ .../TagDataStatisticsResourceFacadeImp.java | 74 +++++++++--- aidr-common/pom.xml | 20 +++- .../aidr/common/code/FasterXmlWrapper.java | 18 +++ 9 files changed, 341 insertions(+), 34 deletions(-) create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountDTO.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountSeriesDTO.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TimeWindowTagCountDTO.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TagCountDTOHelper.java create mode 100644 aidr-common/src/main/java/qa/qcri/aidr/common/code/FasterXmlWrapper.java diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java index e9dd11526..40f28128c 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java @@ -1,6 +1,9 @@ package qa.qcri.aidr.analysis.api; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.ejb.EJB; import javax.json.Json; @@ -22,9 +25,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import qa.qcri.aidr.common.code.DateFormatConfig; +import qa.qcri.aidr.common.code.FasterXmlWrapper; import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.output.getdata.ChannelBufferManager; import qa.qcri.aidr.analysis.api.GetStatistics; +import qa.qcri.aidr.analysis.dto.TagCountDTO; +import qa.qcri.aidr.analysis.dto.helper.TagCountDTOHelper; import qa.qcri.aidr.analysis.entity.TagData; import qa.qcri.aidr.analysis.entity.TagDataPK; import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; @@ -35,19 +41,91 @@ public class GetTagDataStatistics extends GetStatistics implements ServletContex // Debugging private static Logger logger = Logger.getLogger(GetTagDataStatistics.class); private static ErrorLog elog = new ErrorLog(); - + @EJB - private TagDataStatisticsResourceFacade tagDataEJB; + private TagDataStatisticsResourceFacade tagDataEJB; + + @GET + @Path("/getCount/{crisisCode}/{classifierCode}/{granularity}") + @Produces(MediaType.APPLICATION_JSON) + public Response getTagCountFromTime(@PathParam("crisisCode") String crisisCode, + @PathParam("attributeCode") String attributeCode, + @PathParam("granularity") String granularity, + @DefaultValue("0") @QueryParam("startTime") Long startTime) { + + long timeGranularity = DateFormatConfig.parseTime(granularity); + // First get the list of data points from DB + List tagDataList = tagDataEJB.getDataAfterTimestampGranularity(crisisCode, attributeCode, null, startTime, timeGranularity); + + // Now the real work - count and send response + if (tagDataList != null) { + Map tagCountMap = new HashMap(tagDataList.size()); + for (TagData t: tagDataList) { + if (tagCountMap.containsKey(t.getLabelCode())) { + tagCountMap.put(t.getLabelCode(), tagCountMap.get(t.getLabelCode()) + t.getCount()); + } else { + tagCountMap.put(t.getLabelCode(), t.getCount()); + } + } + try { + ObjectMapper mapper = FasterXmlWrapper.getObjectMapper(); + Response.ok(mapper.writeValueAsString(tagCountMap)).build(); + } catch (Exception e) { + logger.info("Error in serializing fetched tag count data"); + e.printStackTrace(); + } + } + return Response.ok(JsonValue.NULL).build(); + } + + @GET + @Path("/getTimeSeries/{crisisCode}/{classifierCode}/{granularity}") + @Produces(MediaType.APPLICATION_JSON) + public Response getTagCountTimeSeries(@PathParam("crisisCode") String crisisCode, + @PathParam("attributeCode") String attributeCode, + @PathParam("granularity") String granularity, + @DefaultValue("0") @QueryParam("startTime") Long startTime, + @QueryParam("endTime") Long endTime) { + if (null == endTime || endTime < startTime) { + endTime = System.currentTimeMillis(); + } + long timeGranularity = DateFormatConfig.parseTime(granularity); + // First get the list of data points from DB + List tagDataList = tagDataEJB.getDataInIntervalWithGranularity(crisisCode, attributeCode, null, startTime, endTime, timeGranularity); + + // Now the real work - format and send response + if (tagDataList != null) { + /* TODO: code for creating time series + Map tagCountMap = new HashMap(tagDataList.size()); + for (TagData t: tagDataList) { + if (tagCountMap.containsKey(t.getLabelCode())) { + tagCountMap.put(t.getLabelCode(), tagCountMap.get(t.getLabelCode()) + t.getCount()); + } else { + tagCountMap.put(t.getLabelCode(), t.getCount()); + } + }*/ + try { + ObjectMapper mapper = FasterXmlWrapper.getObjectMapper(); + //Response.ok(mapper.writeValueAsString(**tagCountMap**)).build(); + } catch (Exception e) { + logger.info("Error in serializing fetched tag count data"); + e.printStackTrace(); + } + } + return Response.ok(JsonValue.NULL).build(); + } + + @GET - @Path("/getTagCount/{crisisCode}/{classifierCode}/{labelCode}") + @Path("/getOneTagCount/{crisisCode}/{classifierCode}/{labelCode}/{granularity}/{startTime}") @Produces(MediaType.APPLICATION_JSON) public Response getSingleItem(@PathParam("crisisCode") String crisisCode, @PathParam("attributeCode") String attributeCode, @PathParam("labelCode") String labelCode, - @DefaultValue("5m") @QueryParam("granularity") String granularity, - @DefaultValue("0") @QueryParam("startTime") Long startTime) { - + @PathParam("granularity") String granularity, + @PathParam("startTime") Long startTime) { + long timeGranularity = DateFormatConfig.parseTime(granularity); TagDataPK tagDataPK = new TagDataPK(); tagDataPK.setCrisisCode(crisisCode); @@ -55,9 +133,9 @@ public Response getSingleItem(@PathParam("crisisCode") String crisisCode, tagDataPK.setGranularity(timeGranularity); tagDataPK.setAttributeCode(attributeCode); tagDataPK.setLabelCode(labelCode); - + TagData obj = tagDataEJB.getSingleDataByPK(tagDataPK); - + List fetchedList = tagDataEJB.getDataByCrisisAttributeLabel(crisisCode, attributeCode, labelCode); ObjectMapper mapper = new ObjectMapper(); try { @@ -68,15 +146,15 @@ public Response getSingleItem(@PathParam("crisisCode") String crisisCode, } return Response.ok(JsonValue.NULL).build(); } - + @GET @Path("/ping") @Produces(MediaType.APPLICATION_JSON) public Response ping() { JsonObject obj = Json.createObjectBuilder() - .add("aidr-analysis/tagData", "RUNNING") - .build(); - + .add("aidr-analysis/tagData", "RUNNING") + .build(); + return Response.ok(obj).build(); } @@ -102,13 +180,13 @@ public Response restartAnalyticsService(@PathParam("passcode") String passcode) @Override public void contextDestroyed(ServletContextEvent arg0) { // TODO Auto-generated method stub - + } @Override public void contextInitialized(ServletContextEvent arg0) { // TODO Auto-generated method stub - + } - + } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java index 23c9fd232..d5c574453 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -37,7 +37,7 @@ public class WriteStatisticsData extends ChannelBufferManager implements ServletContextListener { // Debugging - private static Logger logger = Logger.getLogger(WriteStatisticsData.class); + private static Logger logger = Logger.getLogger(WriteStatisticsData.class.getSuperclass()); private static ErrorLog elog = new ErrorLog(); private volatile boolean runFlag = false; diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountDTO.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountDTO.java new file mode 100644 index 000000000..f164fcdc8 --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountDTO.java @@ -0,0 +1,40 @@ +package qa.qcri.aidr.analysis.dto; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import qa.qcri.aidr.analysis.entity.TagData; + +@XmlRootElement +public class TagCountDTO implements Serializable { + /** + * + */ + private static final long serialVersionUID = 1L; + + @XmlElement + private String labelCode; + + @XmlElement + private Integer count; + + public TagCountDTO() {} + + public String getLabelCode() { + return this.labelCode; + } + + public void setLabelCode(String labelCode) { + this.labelCode = labelCode; + } + + public Integer getCount() { + return this.count; + } + + public void setCount(Integer count) { + this.count = count; + } +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountSeriesDTO.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountSeriesDTO.java new file mode 100644 index 000000000..4fb13d15a --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountSeriesDTO.java @@ -0,0 +1,51 @@ +package qa.qcri.aidr.analysis.dto; + +import java.io.Serializable; +import java.util.List; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class TagCountSeriesDTO implements Serializable { + /** + * + */ + private static final long serialVersionUID = 1L; + + @XmlElement + private String crisisCode; + + @XmlElement + private Long granularity; + + @XmlElement + private List timeSeriesData; + + public TagCountSeriesDTO() {} + + public String getCrisisCode() { + return this.crisisCode; + } + + public void setCrisisCode(String crisisCode) { + this.crisisCode = crisisCode; + } + + public Long getGranularity() { + return this.granularity; + } + + public void setGranularity(Long granularity) { + this.granularity = granularity; + } + + public List getTimeSeriesData() { + return this.timeSeriesData; + } + + public void setTimeSeriesData(List timeSeriesData) { + this.timeSeriesData = timeSeriesData; + } + +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TimeWindowTagCountDTO.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TimeWindowTagCountDTO.java new file mode 100644 index 000000000..8ec8e7c1f --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TimeWindowTagCountDTO.java @@ -0,0 +1,40 @@ +package qa.qcri.aidr.analysis.dto; + +import java.io.Serializable; +import java.util.List; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class TimeWindowTagCountDTO implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 1L; + + @XmlElement + private Long timestamp; + + @XmlElement + private List tagCountData; + + public TimeWindowTagCountDTO() {} + + public Long getTimestamp() { + return this.timestamp; + } + + public void setTimestap(Long timestamp) { + this.timestamp = timestamp; + } + + public List getTagCountData() { + return this.tagCountData; + } + + public void setTagCountData(List tagCountData) { + this.tagCountData = tagCountData; + } +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TagCountDTOHelper.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TagCountDTOHelper.java new file mode 100644 index 000000000..96583e69b --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TagCountDTOHelper.java @@ -0,0 +1,22 @@ +package qa.qcri.aidr.analysis.dto.helper; + +import qa.qcri.aidr.analysis.dto.TagCountDTO; +import qa.qcri.aidr.analysis.entity.TagData; + +public class TagCountDTOHelper { + public static TagCountDTO convertTagDataToDTO(TagData t) { + TagCountDTO dto = new TagCountDTO(); + dto.setLabelCode(t.getLabelCode()); + dto.setCount(t.getCount()); + + return dto; + } + + public static TagData convertDTOToTagData(TagCountDTO dto) { + TagData t = new TagData(); + t.setLabelCode(dto.getLabelCode()); + t.setCount(dto.getCount()); + + return t; + } +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java index ae139dd0f..b4177697c 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java @@ -81,8 +81,12 @@ public List getDataByCrisisAttributeLabel(String crisisCode, String att Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)); + .add(Restrictions.eq("attribute_code", attributeCode)); + if (labelCode != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("label_code", labelCode)); + } criteria.add(criterion); try { List objList = (List) criteria.list(); @@ -117,8 +121,12 @@ public List getDataAfterTimestamp(String crisisCode, String attributeCo Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisis_code", crisisCode)) .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) .add(Restrictions.ge("timestamp", timestamp)); + if (labelCode != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("label_code", labelCode)); + } criteria.add(criterion); try { @@ -137,10 +145,13 @@ public List getDataAfterTimestampGranularity(String crisisCode, String Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisis_code", crisisCode)) .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) .add(Restrictions.ge("timestamp", timestamp)) .add(Restrictions.eq("granularity", granularity)); - + if (labelCode != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("label_code", labelCode)); + } criteria.add(criterion); try { List objList = (List) criteria.list(); @@ -157,8 +168,12 @@ public List getDataBeforeTimestamp(String crisisCode, String attributeC Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisis_code", crisisCode)) .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) .add(Restrictions.le("timestamp", timestamp)); + if (labelCode != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("label_code", labelCode)); + } criteria.add(criterion); try { @@ -177,9 +192,13 @@ public List getDataBeforeTimestampGranularity(String crisisCode, String Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisis_code", crisisCode)) .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) .add(Restrictions.le("timestamp", timestamp)) .add(Restrictions.eq("granularity", granularity)); + if (labelCode != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("label_code", labelCode)); + } criteria.add(criterion); try { @@ -198,10 +217,13 @@ public List getDataInInterval(String crisisCode, String attributeCode, Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisis_code", crisisCode)) .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) .add(Restrictions.ge("timestamp", timestamp1)) .add(Restrictions.le("timestamp", timestamp2)); - + if (labelCode != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("label_code", labelCode)); + } criteria.add(criterion); try { List objList = (List) criteria.list(); @@ -216,14 +238,34 @@ public List getDataInInterval(String crisisCode, String attributeCode, public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2, Long granularity) { Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.ge("timestamp", timestamp1)) - .add(Restrictions.le("timestamp", timestamp2)); + Criterion criterion = Restrictions.eq("crisis_code", crisisCode); + // Now add the optional non-null criteria + if (attributeCode != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("attribute_code", attributeCode)); + } + if (labelCode != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("label_code", labelCode)); + } + if (granularity != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("granularity", granularity)); + } + if (timestamp1 != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.ge("timestamp", timestamp1)); + } + if (timestamp2 != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.le("timestamp", timestamp2)); + } criteria.add(criterion); try { List objList = (List) criteria.list(); diff --git a/aidr-common/pom.xml b/aidr-common/pom.xml index 007957f6f..9e2ea3b1d 100644 --- a/aidr-common/pom.xml +++ b/aidr-common/pom.xml @@ -5,11 +5,11 @@ aidr-common 1.0 - + 1.7 1.9.13 - + log4j @@ -30,6 +30,22 @@ ${jackson-asl.version} true + + + com.fasterxml.jackson.core + jackson-core + 2.4.2 + + + com.fasterxml.jackson.core + jackson-databind + 2.4.2 + + + com.fasterxml.jackson.core + jackson-annotations + 2.4.2 + diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/code/FasterXmlWrapper.java b/aidr-common/src/main/java/qa/qcri/aidr/common/code/FasterXmlWrapper.java new file mode 100644 index 000000000..b6968a01d --- /dev/null +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/code/FasterXmlWrapper.java @@ -0,0 +1,18 @@ +package qa.qcri.aidr.common.code; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + + +public class FasterXmlWrapper { + public static ObjectMapper getObjectMapper() { + try { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + return objectMapper; + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } +} From 64ebbcb18c34a54cb5b1e14dc69583b1287a6772 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Thu, 25 Sep 2014 18:17:13 +0300 Subject: [PATCH 3/6] Implementing the 3 REST APIs in pivotal --- aidr-analytics/pom.xml | 6 +- .../aidr/analysis/api/ApplicationConfig.java | 2 +- ...tics.java => GetConfidenceStatistics.java} | 12 +- .../analysis/api/GetTagDataStatistics.java | 231 +++++-- .../analysis/api/WriteStatisticsData.java | 79 ++- .../aidr/analysis/dto/TagCountSeriesDTO.java | 22 + .../analysis/dto/TimeWindowTagCountDTO.java | 2 +- .../dto/helper/TagCountSeriesDTOHelper.java | 31 + .../helper/TimeWindowTagCountDTOHelper.java | 27 + ...FrequencyData.java => ConfidenceData.java} | 10 +- ...uencyDataPK.java => ConfidenceDataPK.java} | 8 +- .../qa/qcri/aidr/analysis/entity/TagData.java | 4 + .../ConfidenceStatisticsResourceFacade.java | 94 +++ .../FrequencyStatisticsResourceFacade.java | 94 --- .../TagDataStatisticsResourceFacade.java | 3 + ...ConfidenceStatisticsResourceFacadeImp.java | 617 ++++++++++++++++++ .../FrequencyStatisticsResourceFacadeImp.java | 616 ----------------- .../TagDataStatisticsResourceFacadeImp.java | 145 ++-- ...aMapRecord.java => ConfDataMapRecord.java} | 4 +- .../aidr/analysis/utils/CommonOperations.java | 1 + .../aidr/analysis/utils/GranularityData.java | 1 + .../aidr/analysis/utils/JsonResponse.java | 15 + .../main/resources/META-INF/persistence.xml | 6 +- .../src/main/webapp/WEB-INF/web.xml | 2 +- 24 files changed, 1166 insertions(+), 866 deletions(-) rename aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/{GetFrequencyStatistics.java => GetConfidenceStatistics.java} (84%) create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TagCountSeriesDTOHelper.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TimeWindowTagCountDTOHelper.java rename aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/{FrequencyData.java => ConfidenceData.java} (87%) rename aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/{FrequencyDataPK.java => ConfidenceDataPK.java} (91%) create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/ConfidenceStatisticsResourceFacade.java delete mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/FrequencyStatisticsResourceFacade.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java delete mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java rename aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/{FreqDataMapRecord.java => ConfDataMapRecord.java} (92%) create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java diff --git a/aidr-analytics/pom.xml b/aidr-analytics/pom.xml index 29579422c..c9cdc1688 100644 --- a/aidr-analytics/pom.xml +++ b/aidr-analytics/pom.xml @@ -2,11 +2,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 qa.qcri.aidr.analysis - aidr-analysis + aidr-analytics 1.0 war - aidr-analysis - Analysis module for AIDR classifier data + aidr-analytics + Analytics module for AIDR classifier data diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java index c8c33f22c..ccb24d782 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java @@ -33,7 +33,7 @@ private void addRestResourceClasses(Set> resources) { resources.add(MoxyJsonFeature.class); resources.add(JacksonFeature.class); resources.add(qa.qcri.aidr.analysis.api.GetTagDataStatistics.class); - resources.add(qa.qcri.aidr.analysis.api.GetFrequencyStatistics.class); + resources.add(qa.qcri.aidr.analysis.api.GetConfidenceStatistics.class); resources.add(qa.qcri.aidr.analysis.api.WriteStatisticsData.class); } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetFrequencyStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetConfidenceStatistics.java similarity index 84% rename from aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetFrequencyStatistics.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetConfidenceStatistics.java index a61b2213d..a201abb26 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetFrequencyStatistics.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetConfidenceStatistics.java @@ -17,20 +17,20 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.analysis.facade.FrequencyStatisticsResourceFacade; +import qa.qcri.aidr.analysis.facade.ConfidenceStatisticsResourceFacade; import qa.qcri.aidr.common.code.DateFormatConfig; import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.output.getdata.ChannelBufferManager; -@Path("/freqData/") -public class GetFrequencyStatistics extends GetStatistics implements ServletContextListener { +@Path("/confData/") +public class GetConfidenceStatistics extends GetStatistics implements ServletContextListener { // Debugging - private static Logger logger = Logger.getLogger(GetFrequencyStatistics.class); + private static Logger logger = Logger.getLogger(GetConfidenceStatistics.class); private static ErrorLog elog = new ErrorLog(); @EJB - private FrequencyStatisticsResourceFacade freqDataEJB; + private ConfidenceStatisticsResourceFacade confDataEJB; @GET @Path("/getBinCount/{classifierCode}/{labelCode}/{bin}") @@ -70,7 +70,7 @@ public Response restartAnalyticsService(@PathParam("passcode") String passcode) masterCBManager = new ChannelBufferManager(); masterCBManager.initiateChannelBufferManager(CHANNEL_REG_EX); logger.info("aidr-output analytics service restarted..."); - final String statusStr = "{\"aidr-output analytics FrequencyStats service\":\"RESTARTED\"}"; + final String statusStr = "{\"aidr-output analytics ConfidenceStats service\":\"RESTARTED\"}"; return Response.ok(statusStr).build(); } return Response.ok(new String("{\"password\":\"invalid\"}")).build(); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java index 40f28128c..d76c754c8 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java @@ -1,14 +1,16 @@ package qa.qcri.aidr.analysis.api; +import java.util.Date; import java.util.ArrayList; -import java.util.HashMap; + import java.util.List; import java.util.Map; +import java.util.TreeMap; import javax.ejb.EJB; import javax.json.Json; import javax.json.JsonObject; -import javax.json.JsonValue; + import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; import javax.ws.rs.DefaultValue; @@ -20,20 +22,25 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import net.minidev.json.JSONObject; + import org.apache.log4j.Logger; import com.fasterxml.jackson.databind.ObjectMapper; -import qa.qcri.aidr.common.code.DateFormatConfig; import qa.qcri.aidr.common.code.FasterXmlWrapper; import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.output.getdata.ChannelBufferManager; -import qa.qcri.aidr.analysis.api.GetStatistics; import qa.qcri.aidr.analysis.dto.TagCountDTO; +import qa.qcri.aidr.analysis.dto.TagCountSeriesDTO; +import qa.qcri.aidr.analysis.dto.TimeWindowTagCountDTO; import qa.qcri.aidr.analysis.dto.helper.TagCountDTOHelper; +import qa.qcri.aidr.analysis.dto.helper.TagCountSeriesDTOHelper; +import qa.qcri.aidr.analysis.dto.helper.TimeWindowTagCountDTOHelper; import qa.qcri.aidr.analysis.entity.TagData; import qa.qcri.aidr.analysis.entity.TagDataPK; import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; +import qa.qcri.aidr.analysis.utils.JsonResponse; + @Path("/tagData/") public class GetTagDataStatistics extends GetStatistics implements ServletContextListener { @@ -45,21 +52,31 @@ public class GetTagDataStatistics extends GetStatistics implements ServletContex @EJB private TagDataStatisticsResourceFacade tagDataEJB; + /** + * + * @param crisisCode + * @param attributeCode + * @param granularity + * @param startTime + * @return Count sum per label from startTime to current time at the given granularity, for crisisCode and attributeCode + */ @GET - @Path("/getCount/{crisisCode}/{classifierCode}/{granularity}") + @Path("/getLabelSum/{crisisCode}/{classifierCode}/{granularity}") @Produces(MediaType.APPLICATION_JSON) - public Response getTagCountFromTime(@PathParam("crisisCode") String crisisCode, + public Response getTagCountSumFromTime(@PathParam("crisisCode") String crisisCode, @PathParam("attributeCode") String attributeCode, - @PathParam("granularity") String granularity, + @PathParam("granularity") Long granularity, @DefaultValue("0") @QueryParam("startTime") Long startTime) { - long timeGranularity = DateFormatConfig.parseTime(granularity); + //long timeGranularity = DateFormatConfig.parseTime(granularity); // First get the list of data points from DB - List tagDataList = tagDataEJB.getDataAfterTimestampGranularity(crisisCode, attributeCode, null, startTime, timeGranularity); + List tagDataList = tagDataEJB.getDataAfterTimestampGranularity(crisisCode, attributeCode, null, startTime, granularity); // Now the real work - count and send response + JSONObject json = JsonResponse.getNewJsonResponseObject(crisisCode, attributeCode, granularity); + json.put("startTime", new Date(startTime)); if (tagDataList != null) { - Map tagCountMap = new HashMap(tagDataList.size()); + Map tagCountMap = new TreeMap(); for (TagData t: tagDataList) { if (tagCountMap.containsKey(t.getLabelCode())) { tagCountMap.put(t.getLabelCode(), tagCountMap.get(t.getLabelCode()) + t.getCount()); @@ -68,83 +85,210 @@ public Response getTagCountFromTime(@PathParam("crisisCode") String crisisCode, } } try { - ObjectMapper mapper = FasterXmlWrapper.getObjectMapper(); - Response.ok(mapper.writeValueAsString(tagCountMap)).build(); + json.put("data", tagCountMap); + } catch (Exception e) { + logger.info("Error in serializing fetched tag count data"); + e.printStackTrace(); + } + } + return Response.ok(json.toJSONString()).build(); + } + + /** + * + * @param crisisCode + * @param attributeCode + * @param granularity + * @param startTime + * @return Count per label in the specified time window at the given granularity, for crisisCode and attributeCode + */ + @GET + @Path("/getLabelCount/{crisisCode}/{attributeCode}/{granularity}") + @Produces(MediaType.APPLICATION_JSON) + public Response getTagCountInTimeWindow(@PathParam("crisisCode") String crisisCode, + @PathParam("attributeCode") String attributeCode, + @PathParam("granularity") Long granularity, + @DefaultValue("0") @QueryParam("startTime") Long startTime) { + + //long timeGranularity = DateFormatConfig.parseTime(granularity); + // First get the list of data points from DB + List tagDataList = tagDataEJB.getDataByGranularityInTimeWindow(crisisCode, attributeCode, null, startTime, granularity); + + // Now the real work - count and send response + TimeWindowTagCountDTO dto = new TimeWindowTagCountDTO(); + JSONObject json = JsonResponse.getNewJsonResponseObject(crisisCode, attributeCode, granularity); + json.put("timestamp", new Date(startTime)); + + if (tagDataList != null) { + try { + List dtoList = new ArrayList(); + for (TagData t: tagDataList) { + dtoList.add(TagCountDTOHelper.convertTagDataToDTO(t)); + System.out.println("tag: " + t.getLabelCode() + ", count: " + t.getCount()); + } + dto = TimeWindowTagCountDTOHelper.convertTagCountDTOListToDTO(startTime, dtoList); + json.put("data", dto); } catch (Exception e) { logger.info("Error in serializing fetched tag count data"); e.printStackTrace(); } } - return Response.ok(JsonValue.NULL).build(); + return Response.ok(json.toJSONString()).build(); } + /** + * + * @param crisisCode + * @param attributeCode + * @param granularity + * @param startTime + * @param endTime + * @return Time series data for each label in the interval [startTime, endTime] at the given granularity, for crisisCode and attributeCode + */ @GET - @Path("/getTimeSeries/{crisisCode}/{classifierCode}/{granularity}") + @Path("/getLabelTimeSeries/{crisisCode}/{attributeCode}/{granularity}") @Produces(MediaType.APPLICATION_JSON) public Response getTagCountTimeSeries(@PathParam("crisisCode") String crisisCode, @PathParam("attributeCode") String attributeCode, - @PathParam("granularity") String granularity, + @PathParam("granularity") Long granularity, @DefaultValue("0") @QueryParam("startTime") Long startTime, @QueryParam("endTime") Long endTime) { - + if (null == endTime || endTime < startTime) { endTime = System.currentTimeMillis(); } - long timeGranularity = DateFormatConfig.parseTime(granularity); + //long timeGranularity = DateFormatConfig.parseTime(granularity); // First get the list of data points from DB - List tagDataList = tagDataEJB.getDataInIntervalWithGranularity(crisisCode, attributeCode, null, startTime, endTime, timeGranularity); + List tagDataList = tagDataEJB.getDataInIntervalWithGranularity(crisisCode, attributeCode, null, startTime, endTime, granularity); - // Now the real work - format and send response + // Now the real work - creat time series, format and send response + JSONObject json = new JSONObject(); if (tagDataList != null) { - /* TODO: code for creating time series - Map tagCountMap = new HashMap(tagDataList.size()); + try { + // Create time series Map data first + Map> tagCountMap = new TreeMap>(); + for (TagData t: tagDataList) { + if (tagCountMap.containsKey(t.getTimestamp())) { + List tagsList = tagCountMap.get(t.getTimestamp()); + if (null == tagsList || tagsList.isEmpty()) { + tagsList = new ArrayList(); + tagCountMap.put(t.getTimestamp(), tagsList); + } + tagsList.add(TagCountDTOHelper.convertTagDataToDTO(t)); + } else { + tagCountMap.put(t.getTimestamp(), new ArrayList()); + List tagsList = tagCountMap.get(t.getTimestamp()); + tagsList.add(TagCountDTOHelper.convertTagDataToDTO(t)); + } + } + //System.out.println("Finished creating Map of timestamp versus TagCountDTO list"); + // Now convert the above time series data Map to DTO object for response + List timeWindowDTOList = new ArrayList(); + for (Long key: tagCountMap.keySet()) { + TimeWindowTagCountDTO timeWindowDTO = TimeWindowTagCountDTOHelper.convertTagCountDTOListToDTO(key, tagCountMap.get(key)); + timeWindowDTOList.add(timeWindowDTO); + } + //System.out.println("Finished creating TimeWindowTagCountDTO list"); + TagCountSeriesDTO dto = TagCountSeriesDTOHelper.convertTimeWindowTagCountDTOListToDTO(crisisCode, attributeCode, granularity, timeWindowDTOList); + //System.out.println("Finished creating TagCountSeriesDTO"); + ObjectMapper mapper = FasterXmlWrapper.getObjectMapper(); + //json.put("time_series", dto); + //System.out.println("Finished creating json object: " + json); + return Response.ok(mapper.writeValueAsString(dto)).build(); + } catch (Exception e) { + logger.info("Error in serializing fetched tag count data"); + e.printStackTrace(); + } + } + return Response.ok(JsonResponse.getNewJsonResponseObject(crisisCode, attributeCode, granularity).toJSONString()).build(); + } + + /** + * + * @param crisisCode + * @param attributeCode + * @param granularity + * @param startTime + * @param endTime + * @return Count sum for each label in the interval [startTime, endTime] at the given granularity, for crisisCode and attributeCode + */ + @GET + @Path("/getIntervalLabelSum/{crisisCode}/{attributeCode}/{granularity}") + @Produces(MediaType.APPLICATION_JSON) + public Response getTagCountSumInInterval(@PathParam("crisisCode") String crisisCode, + @PathParam("attributeCode") String attributeCode, + @PathParam("granularity") Long granularity, + @DefaultValue("0") @QueryParam("startTime") Long startTime, + @QueryParam("endTime") Long endTime) { + + //long timeGranularity = DateFormatConfig.parseTime(granularity); + // First get the list of data points from DB + List tagDataList = tagDataEJB.getDataInIntervalWithGranularity(crisisCode, attributeCode, null, startTime, endTime, granularity); + + // Now the real work - count and send response + JSONObject json = JsonResponse.getNewJsonResponseObject(crisisCode, attributeCode, granularity); + json.put("startTime", new Date(startTime)); + json.put("endTime", new Date(endTime)); + + if (tagDataList != null) { + Map tagCountMap = new TreeMap(); for (TagData t: tagDataList) { + System.out.println("Looking at tag: " + t.getLabelCode() + ", having count = " + t.getCount()); if (tagCountMap.containsKey(t.getLabelCode())) { tagCountMap.put(t.getLabelCode(), tagCountMap.get(t.getLabelCode()) + t.getCount()); + System.out.println("Update Map for OLD tag = " + t.getLabelCode() + " with count = " + tagCountMap.get(t.getLabelCode())); } else { tagCountMap.put(t.getLabelCode(), t.getCount()); + System.out.println("Update Map with NEW tag = " + t.getLabelCode() + " with count = " + tagCountMap.get(t.getLabelCode())); } - }*/ + } try { - ObjectMapper mapper = FasterXmlWrapper.getObjectMapper(); - //Response.ok(mapper.writeValueAsString(**tagCountMap**)).build(); + json.put("data", tagCountMap); } catch (Exception e) { logger.info("Error in serializing fetched tag count data"); e.printStackTrace(); } } - return Response.ok(JsonValue.NULL).build(); + return Response.ok(json.toJSONString()).build(); } + /** + * + * @param crisisCode + * @param attributeCode + * @param labelCode + * @param granularity + * @param startTime + * @return The count for a label in the specified time window at the given granularity, for crisisCode and attributeCode + */ @GET - @Path("/getOneTagCount/{crisisCode}/{classifierCode}/{labelCode}/{granularity}/{startTime}") + @Path("/getOneLabelData/{crisisCode}/{attributeCode}/{labelCode}/{granularity}/{startTime}") @Produces(MediaType.APPLICATION_JSON) public Response getSingleItem(@PathParam("crisisCode") String crisisCode, @PathParam("attributeCode") String attributeCode, @PathParam("labelCode") String labelCode, - @PathParam("granularity") String granularity, + @PathParam("granularity") Long granularity, @PathParam("startTime") Long startTime) { - long timeGranularity = DateFormatConfig.parseTime(granularity); + //long timeGranularity = DateFormatConfig.parseTime(granularity); TagDataPK tagDataPK = new TagDataPK(); tagDataPK.setCrisisCode(crisisCode); tagDataPK.setTimestamp(startTime); - tagDataPK.setGranularity(timeGranularity); + tagDataPK.setGranularity(granularity); tagDataPK.setAttributeCode(attributeCode); tagDataPK.setLabelCode(labelCode); TagData obj = tagDataEJB.getSingleDataByPK(tagDataPK); - - List fetchedList = tagDataEJB.getDataByCrisisAttributeLabel(crisisCode, attributeCode, labelCode); - ObjectMapper mapper = new ObjectMapper(); + JSONObject json = JsonResponse.getNewJsonResponseObject(crisisCode, attributeCode, granularity); try { - return Response.ok(mapper.writeValueAsString(fetchedList)).build(); + json.put("timestamp", new Date(startTime)); + json.put("data", obj); } catch (Exception e) { logger.error("Serialization error"); logger.error(elog.toStringException(e)); } - return Response.ok(JsonValue.NULL).build(); + return Response.ok(json.toJSONString()).build(); } @GET @@ -159,23 +303,6 @@ public Response ping() { } - @GET - @Path("/manage/restart/{passcode}") - @Produces("application/json") - public Response restartAnalyticsService(@PathParam("passcode") String passcode) { - logger.info("Request received"); - if (passcode.equals("sysadmin2013")) { - if (masterCBManager != null) { - masterCBManager.close(); - } - masterCBManager = new ChannelBufferManager(); - masterCBManager.initiateChannelBufferManager(CHANNEL_REG_EX); - logger.info("aidr-output analytics service restarted..."); - final String statusStr = "{\"aidr-output analytics TagStats service\":\"RESTARTED\"}"; - return Response.ok(statusStr).build(); - } - return Response.ok(new String("{\"password\":\"invalid\"}")).build(); - } @Override public void contextDestroyed(ServletContextEvent arg0) { diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java index d5c574453..f80c02f64 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -4,6 +4,7 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -19,13 +20,15 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import net.minidev.json.JSONObject; + import org.apache.log4j.Logger; -import qa.qcri.aidr.analysis.entity.FrequencyData; +import qa.qcri.aidr.analysis.entity.ConfidenceData; import qa.qcri.aidr.analysis.entity.TagData; import qa.qcri.aidr.analysis.stat.*; import qa.qcri.aidr.analysis.utils.GranularityData; -import qa.qcri.aidr.analysis.facade.FrequencyStatisticsResourceFacade; +import qa.qcri.aidr.analysis.facade.ConfidenceStatisticsResourceFacade; import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.values.ReturnCode; @@ -46,25 +49,25 @@ public class WriteStatisticsData extends ChannelBufferManager implements Servlet private TagDataStatisticsResourceFacade tagDataEJB; @EJB - private FrequencyStatisticsResourceFacade freqDataEJB; + private ConfidenceStatisticsResourceFacade confDataEJB; private ExecutorService executorServicePool = null; private Thread t = null; private static final String SENTINEL = "#"; private static ConcurrentHashMap tagDataMap = null; - private static ConcurrentHashMap freqDataMap = null; + private static ConcurrentHashMap confDataMap = null; private static ConcurrentHashMap channelMap = null; private List granularityList = null; private long lastTagDataCheckedTime = 0; - private long lastFreqDataCheckedTime = 0; + private long lastConfDataCheckedTime = 0; @Override public void contextInitialized(ServletContextEvent sce) { logger.info("Initializing channel buffer manager"); System.out.println("[contextInitialized] Initializing channel buffer manager"); tagDataMap = new ConcurrentHashMap(); - freqDataMap = new ConcurrentHashMap(); + confDataMap = new ConcurrentHashMap(); channelMap = new ConcurrentHashMap(); granularityList = GranularityData.getGranularities(); @@ -86,12 +89,22 @@ public void contextInitialized(ServletContextEvent sce) { @Override public void contextDestroyed(ServletContextEvent sce) { tagDataMap.clear(); - freqDataMap.clear(); + confDataMap.clear(); runFlag = false; close(); logger.info("Context destroyed"); } - + + @GET + @Path("/tracked/channels") + @Produces(MediaType.APPLICATION_JSON) + public Response getTrackedChannelsList() { + Set channelList = getActiveChannelsList(); + JSONObject json = new JSONObject(); + json.put("channels", channelList); + return Response.ok(json.toJSONString()).build(); + } + @GET @Path("/ping") @Produces(MediaType.APPLICATION_JSON) @@ -104,7 +117,7 @@ public Response ping() { } private ReturnCode writeOutputDataToTagDataDB(Long granularity, Long timestamp) { - System.out.println("tagDataMap size: " + (tagDataMap != null ? tagDataMap.size() : "null")); + //System.out.println("tagDataMap size: " + (tagDataMap != null ? tagDataMap.size() : "null")); try { for (String key: tagDataMap.keySet()) { String[] data = key.split(SENTINEL); @@ -118,7 +131,7 @@ private ReturnCode writeOutputDataToTagDataDB(Long granularity, Long timestamp) tagDataEJB.writeData(t); tCount.resetCount(granularity); TagDataMapRecord temp = (TagDataMapRecord) tagDataMap.get(key); - System.out.println("[writeOutputDataToTagDataDB] After reset, count for key = " + key + " is = " + temp.getCount(granularity)); + //System.out.println("[writeOutputDataToTagDataDB] After reset, count for key = " + key + " is = " + temp.getCount(granularity)); } } catch (Exception e) { @@ -129,23 +142,23 @@ private ReturnCode writeOutputDataToTagDataDB(Long granularity, Long timestamp) return ReturnCode.SUCCESS; } - private ReturnCode writeOutputDataToFreqDataDB(Long granularity, Long timestamp) { + private ReturnCode writeOutputDataToConfDataDB(Long granularity, Long timestamp) { try { - for (String key: freqDataMap.keySet()) { + for (String key: confDataMap.keySet()) { String[] data = key.split(SENTINEL); // data[0] = channelName // data[1] = attributeCode // data[2] = labelCode // data[3] = bin - FreqDataMapRecord fCount = (FreqDataMapRecord) freqDataMap.get(key); - FrequencyData f = new FrequencyData(data[0], timestamp, granularity, data[1], data[2], Integer.parseInt(data[3]), fCount.getCount(granularity)); - freqDataEJB.writeData(f); + ConfDataMapRecord fCount = (ConfDataMapRecord) confDataMap.get(key); + ConfidenceData f = new ConfidenceData(data[0], timestamp, granularity, data[1], data[2], Integer.parseInt(data[3]), fCount.getCount(granularity)); + confDataEJB.writeData(f); fCount.resetCount(granularity); - FreqDataMapRecord temp = (FreqDataMapRecord) freqDataMap.get(key); - System.out.println("[writeOutputDataToFreqDataDB] After reset, count for key = " + key + " is = " + temp.getCount(granularity)); + ConfDataMapRecord temp = (ConfDataMapRecord) confDataMap.get(key); + //System.out.println("[writeOutputDataToConfDataDB] After reset, count for key = " + key + " is = " + temp.getCount(granularity)); } } catch (Exception e) { - System.err.println("[writeOutputDataToFreqDataDB] Error in writing to FrequencyDataDB table!"); + System.err.println("[writeOutputDataToConfDataDB] Error in writing to ConfidenceDataDB table!"); e.printStackTrace(); return ReturnCode.FAIL; } @@ -188,7 +201,7 @@ public void manageChannelBuffersWrapper(final String subscriptionPattern, final .append(nb.attribute_code).append(SENTINEL) .append(nb.label_code); String tagDataKey = new String(keyPrefix); - String freqDataKey = new String(keyPrefix.append(SENTINEL).append(getBinNumber(nb.confidence))); + String confDataKey = new String(keyPrefix.append(SENTINEL).append(getBinNumber(nb.confidence))); if (tagDataMap.containsKey(tagDataKey)) { TagDataMapRecord t = (TagDataMapRecord) tagDataMap.get(tagDataKey); t.incrementAllCounts(); @@ -198,14 +211,14 @@ public void manageChannelBuffersWrapper(final String subscriptionPattern, final tagDataMap.put(tagDataKey, t); System.out.println("[manageChannelBuffersWrapper] New Tag map entry with key: " + tagDataKey + " value = " + tagDataMap.get(tagDataKey)); } - if (freqDataMap.containsKey(freqDataKey)) { - FreqDataMapRecord f = (FreqDataMapRecord) freqDataMap.get(freqDataKey); + if (confDataMap.containsKey(confDataKey)) { + ConfDataMapRecord f = (ConfDataMapRecord) confDataMap.get(confDataKey); f.incrementAllCounts(); - freqDataMap.put(freqDataKey, f); + confDataMap.put(confDataKey, f); } else { - FreqDataMapRecord t = new FreqDataMapRecord(granularityList); - freqDataMap.put(freqDataKey, t); - System.out.println("[manageChannelBuffersWrapper] New Freq map entry with key: " + freqDataKey + " value = " + freqDataMap.get(freqDataKey)); + ConfDataMapRecord t = new ConfDataMapRecord(granularityList); + confDataMap.put(confDataKey, t); + System.out.println("[manageChannelBuffersWrapper] New Conf map entry with key: " + confDataKey + " value = " + confDataMap.get(confDataKey)); } } } @@ -215,7 +228,7 @@ public void manageChannelBuffersWrapper(final String subscriptionPattern, final } // Periodically check if any channel is down - if so, delete all in-memory data for that channel lastTagDataCheckedTime = periodicInactiveChannelCheck(lastTagDataCheckedTime, tagDataMap); - lastFreqDataCheckedTime = periodicInactiveChannelCheck(lastFreqDataCheckedTime, freqDataMap); + lastConfDataCheckedTime = periodicInactiveChannelCheck(lastConfDataCheckedTime, confDataMap); } private long periodicInactiveChannelCheck(long lastCheckedTime, ConcurrentHashMap dataMap) { @@ -256,10 +269,10 @@ private class WriterThread implements Runnable { public void run() { System.out.println("Started writer thread: " + t.getName()); Map lastTagDataWriteTime = new TreeMap(); - Map lastFreqDataWriteTime = new TreeMap(); + Map lastConfDataWriteTime = new TreeMap(); for (Long g: granularityList) { lastTagDataWriteTime.put(g, 0L); - lastFreqDataWriteTime.put(g, 0L); + lastConfDataWriteTime.put(g, 0L); } while (runFlag) { long currentTime = System.currentTimeMillis(); @@ -274,13 +287,13 @@ public void run() { System.out.println("[ToTagDataDB] Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); } } - if (0 == lastFreqDataWriteTime.get(granularity) || (currentTime - lastFreqDataWriteTime.get(granularity)) >= granularity) { + if (0 == lastConfDataWriteTime.get(granularity) || (currentTime - lastConfDataWriteTime.get(granularity)) >= granularity) { // Write to DB table - ReturnCode retVal = writeOutputDataToFreqDataDB(granularity, currentTime); - System.out.println("[ToFreqDataDB] retVal = " + retVal); - lastFreqDataWriteTime.put(granularity, currentTime); + ReturnCode retVal = writeOutputDataToConfDataDB(granularity, currentTime); + System.out.println("[ToConfDataDB] retVal = " + retVal); + lastConfDataWriteTime.put(granularity, currentTime); if (ReturnCode.SUCCESS.equals(retVal)) { - System.out.println("[ToFreqDataDB] Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); + System.out.println("[ToConfDataDB] Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); } } } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountSeriesDTO.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountSeriesDTO.java index 4fb13d15a..7490dee1a 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountSeriesDTO.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TagCountSeriesDTO.java @@ -16,9 +16,15 @@ public class TagCountSeriesDTO implements Serializable { @XmlElement private String crisisCode; + @XmlElement + private String attributeCode; + @XmlElement private Long granularity; + @XmlElement + private Integer total; + @XmlElement private List timeSeriesData; @@ -32,6 +38,14 @@ public void setCrisisCode(String crisisCode) { this.crisisCode = crisisCode; } + public String getAttributeCode() { + return this.attributeCode; + } + + public void setAttributeCode(String attributeCode) { + this.attributeCode = attributeCode; + } + public Long getGranularity() { return this.granularity; } @@ -40,6 +54,14 @@ public void setGranularity(Long granularity) { this.granularity = granularity; } + public Integer getTotal() { + return this.total; + } + + public void setTotal(Integer total) { + this.total = total; + } + public List getTimeSeriesData() { return this.timeSeriesData; } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TimeWindowTagCountDTO.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TimeWindowTagCountDTO.java index 8ec8e7c1f..847902740 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TimeWindowTagCountDTO.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/TimeWindowTagCountDTO.java @@ -26,7 +26,7 @@ public Long getTimestamp() { return this.timestamp; } - public void setTimestap(Long timestamp) { + public void setTimestamp(Long timestamp) { this.timestamp = timestamp; } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TagCountSeriesDTOHelper.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TagCountSeriesDTOHelper.java new file mode 100644 index 000000000..fdac8b30c --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TagCountSeriesDTOHelper.java @@ -0,0 +1,31 @@ +package qa.qcri.aidr.analysis.dto.helper; + +import java.util.ArrayList; +import java.util.List; + +import qa.qcri.aidr.analysis.dto.TagCountSeriesDTO; +import qa.qcri.aidr.analysis.dto.TimeWindowTagCountDTO; + +public class TagCountSeriesDTOHelper { + public static TagCountSeriesDTO convertTimeWindowTagCountDTOListToDTO(String crisisCode, String attributeCode, + Long granularity, List tList) { + TagCountSeriesDTO dto = new TagCountSeriesDTO (); + dto.setCrisisCode(crisisCode); + dto.setAttributeCode(attributeCode); + dto.setGranularity(granularity); + dto.setTimeSeriesData(tList); + dto.setTotal(tList.size()); + + return dto; + } + + public static List convertDTOToTimeWindowTagCountDTOList(TagCountSeriesDTO dto) { + List tList = new ArrayList(); + if (dto.getTimeSeriesData() != null) { + for (TimeWindowTagCountDTO t: dto.getTimeSeriesData()) { + tList.add(t); + } + } + return tList; + } +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TimeWindowTagCountDTOHelper.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TimeWindowTagCountDTOHelper.java new file mode 100644 index 000000000..4eed7e1ef --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/dto/helper/TimeWindowTagCountDTOHelper.java @@ -0,0 +1,27 @@ +package qa.qcri.aidr.analysis.dto.helper; + +import java.util.ArrayList; +import java.util.List; + +import qa.qcri.aidr.analysis.dto.TagCountDTO; +import qa.qcri.aidr.analysis.dto.TimeWindowTagCountDTO; + + +public class TimeWindowTagCountDTOHelper { + public static TimeWindowTagCountDTO convertTagCountDTOListToDTO(long timestamp, List tList) { + TimeWindowTagCountDTO dto = new TimeWindowTagCountDTO(); + dto.setTimestamp(timestamp); + dto.setTagCountData(tList); + return dto; + } + + public static List convertDTOToTagCountDTOList(TimeWindowTagCountDTO dto) { + List tList = new ArrayList(); + if (dto.getTagCountData() != null) { + for (TagCountDTO t: dto.getTagCountData()) { + tList.add(t); + } + } + return tList; + } +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/ConfidenceData.java similarity index 87% rename from aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyData.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/ConfidenceData.java index 89d60146d..9073df425 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/ConfidenceData.java @@ -12,10 +12,10 @@ @Entity -@Table(catalog = "aidr_analysis",name = "freq_data") -@IdClass(value=FrequencyDataPK.class) +@Table(catalog = "aidr_analysis",name = "conf_data") +@IdClass(value=ConfidenceDataPK.class) @XmlRootElement -public class FrequencyData implements Serializable { +public class ConfidenceData implements Serializable { @XmlElement @Id @@ -52,9 +52,9 @@ public class FrequencyData implements Serializable { @Column(name="count", nullable=false) private Integer count; - public FrequencyData() {} + public ConfidenceData() {} - public FrequencyData(String crisisCode, Long timestamp, Long granularity, String attributeCode, String labelCode, Integer bin, Integer count) { + public ConfidenceData(String crisisCode, Long timestamp, Long granularity, String attributeCode, String labelCode, Integer bin, Integer count) { this.crisisCode = crisisCode; this.timestamp = timestamp; this.granularity = granularity; diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyDataPK.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/ConfidenceDataPK.java similarity index 91% rename from aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyDataPK.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/ConfidenceDataPK.java index b85130b34..b18f753c1 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/FrequencyDataPK.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/ConfidenceDataPK.java @@ -2,7 +2,7 @@ import java.io.Serializable; -public class FrequencyDataPK implements Serializable { +public class ConfidenceDataPK implements Serializable { /** * @@ -15,7 +15,7 @@ public class FrequencyDataPK implements Serializable { private String labelCode; private Integer bin; - public FrequencyDataPK() {} + public ConfidenceDataPK() {} public String getCrisisCode() { return this.crisisCode; @@ -67,8 +67,8 @@ public void setBin(Integer bin) { @Override public boolean equals(Object obj) { - if (obj instanceof FrequencyDataPK) { - FrequencyDataPK frequencyDataPK = (FrequencyDataPK) obj; + if (obj instanceof ConfidenceDataPK) { + ConfidenceDataPK frequencyDataPK = (ConfidenceDataPK) obj; if (!frequencyDataPK.getCrisisCode().equals(crisisCode)) return false; if (frequencyDataPK.getTimestamp() != timestamp) return false; if (frequencyDataPK.getGranularity() != granularity) return false; diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java index ad593a761..9297bb12f 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/entity/TagData.java @@ -47,9 +47,11 @@ public class TagData implements Serializable { @Column(name="label_code", nullable=false) private String labelCode; + /* @XmlElement @Column(name="tag_text", nullable=true) private String tagText; + */ @XmlElement @Column(name="count", nullable=false) @@ -115,6 +117,7 @@ public void setLabelCode(String labelCode) { this.labelCode = labelCode; } + /* public String getTagText() { return this.tagText; } @@ -122,6 +125,7 @@ public String getTagText() { public void setTagText(String tagText) { this.tagText = tagText; } + */ public Integer getCount() { return this.count; diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/ConfidenceStatisticsResourceFacade.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/ConfidenceStatisticsResourceFacade.java new file mode 100644 index 000000000..42737e41a --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/ConfidenceStatisticsResourceFacade.java @@ -0,0 +1,94 @@ +package qa.qcri.aidr.analysis.facade; + +import java.util.List; + +import javax.ejb.Local; + +import qa.qcri.aidr.analysis.entity.ConfidenceData; +import qa.qcri.aidr.analysis.entity.ConfidenceDataPK; +import qa.qcri.aidr.analysis.entity.TagData; +import qa.qcri.aidr.common.values.ReturnCode; + + +@Local +public interface ConfidenceStatisticsResourceFacade { + public ReturnCode writeData(ConfidenceData freqData); + + public ConfidenceData getSingleDataByPK(ConfidenceDataPK freqDataPK); + + public List getDataByCrisis(String crisisCode); + + public List getDataByCrisisAttributeLabel(String crisisCode, String attributeCode, String labelCode); + + public List getDataByCrisisAttributeLabelGranularity(String crisisCode, String attributeCode, String labelCode, Long granularity); + + public List getDataAfterTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp); + + public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity); + + public List getDataBeforeTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp); + + public List getDataBeforeTimestampGranularity(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity); + + + public List getDataInInterval(String crisisCode, String attributeCode, String labelCode, + Long timestamp1, Long timestamp2); + + public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, + Long timestamp1, Long timestamp2, Long granularity); + + //////////////////////////////////////////////// + // Bin related + /////////////////////////////////////////////// + public List getDataByCrisisWithBin(String crisisCode, Integer bin); + + public List getDataByCrisisAttributeLabelWithBin(String crisisCode, String attributeCode, String labelCode, Integer bin); + + public List getDataByCrisisAttributeLabelGranularityWithBin(String crisisCode, String attributeCode, String labelCode, + Long granularity, Integer bin); + + public List getDataAfterTimestampWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Integer bin); + + public List getDataAfterTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity, Integer bin); + + public List getDataBeforeTimestampWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Integer bin); + + public List getDataBeforeTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity, Integer bin); + + + public List getDataInIntervalWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp1, Long timestamp2, Integer bin); + + public List getDataInIntervalWithGranularityWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp1, Long timestamp2, Long granularity, Integer bin); + + public List getDataByCrisisInBin(String crisisCode, Integer bin); + + public List getDataByCrisisAttributeLabelInBin(String crisisCode, String attributeCode, String labelCode, Integer bin); + + public List getDataByCrisisAttributeLabelGranularityInBin(String crisisCode, String attributeCode, String labelCode, + Long granularity, Integer bin); + + public List getDataAfterTimestampInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Integer bin); + + public List getDataAfterTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity, Integer bin); + + public List getDataBeforeTimestampInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Integer bin); + + public List getDataBeforeTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity, Integer bin); + + + public List getDataInIntervalInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp1, Long timestamp2, Integer bin); + + public List getDataInIntervalWithGranularityInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp1, Long timestamp2, Long granularity, Integer bin); +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/FrequencyStatisticsResourceFacade.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/FrequencyStatisticsResourceFacade.java deleted file mode 100644 index 7983beef5..000000000 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/FrequencyStatisticsResourceFacade.java +++ /dev/null @@ -1,94 +0,0 @@ -package qa.qcri.aidr.analysis.facade; - -import java.util.List; - -import javax.ejb.Local; - -import qa.qcri.aidr.analysis.entity.FrequencyData; -import qa.qcri.aidr.analysis.entity.FrequencyDataPK; -import qa.qcri.aidr.analysis.entity.TagData; -import qa.qcri.aidr.common.values.ReturnCode; - - -@Local -public interface FrequencyStatisticsResourceFacade { - public ReturnCode writeData(FrequencyData freqData); - - public FrequencyData getSingleDataByPK(FrequencyDataPK freqDataPK); - - public List getDataByCrisis(String crisisCode); - - public List getDataByCrisisAttributeLabel(String crisisCode, String attributeCode, String labelCode); - - public List getDataByCrisisAttributeLabelGranularity(String crisisCode, String attributeCode, String labelCode, Long granularity); - - public List getDataAfterTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp); - - public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity); - - public List getDataBeforeTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp); - - public List getDataBeforeTimestampGranularity(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity); - - - public List getDataInInterval(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2); - - public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2, Long granularity); - - //////////////////////////////////////////////// - // Bin related - /////////////////////////////////////////////// - public List getDataByCrisisWithBin(String crisisCode, Integer bin); - - public List getDataByCrisisAttributeLabelWithBin(String crisisCode, String attributeCode, String labelCode, Integer bin); - - public List getDataByCrisisAttributeLabelGranularityWithBin(String crisisCode, String attributeCode, String labelCode, - Long granularity, Integer bin); - - public List getDataAfterTimestampWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Integer bin); - - public List getDataAfterTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity, Integer bin); - - public List getDataBeforeTimestampWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Integer bin); - - public List getDataBeforeTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity, Integer bin); - - - public List getDataInIntervalWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2, Integer bin); - - public List getDataInIntervalWithGranularityWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2, Long granularity, Integer bin); - - public List getDataByCrisisInBin(String crisisCode, Integer bin); - - public List getDataByCrisisAttributeLabelInBin(String crisisCode, String attributeCode, String labelCode, Integer bin); - - public List getDataByCrisisAttributeLabelGranularityInBin(String crisisCode, String attributeCode, String labelCode, - Long granularity, Integer bin); - - public List getDataAfterTimestampInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Integer bin); - - public List getDataAfterTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity, Integer bin); - - public List getDataBeforeTimestampInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Integer bin); - - public List getDataBeforeTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity, Integer bin); - - - public List getDataInIntervalInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2, Integer bin); - - public List getDataInIntervalWithGranularityInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2, Long granularity, Integer bin); -} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java index 4a051c37c..c79efa6ad 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java @@ -8,6 +8,7 @@ import qa.qcri.aidr.analysis.entity.TagDataPK; import qa.qcri.aidr.common.values.ReturnCode; + @Local public interface TagDataStatisticsResourceFacade { public ReturnCode writeData(TagData tagData); @@ -20,6 +21,8 @@ public interface TagDataStatisticsResourceFacade { public List getDataByCrisisAttributeLabelGranularity(String crisisCode, String attributeCode, String labelCode, Long granularity); + public List getDataByGranularityInTimeWindow(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity); + public List getDataAfterTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp); public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java new file mode 100644 index 000000000..61033a58f --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java @@ -0,0 +1,617 @@ +package qa.qcri.aidr.analysis.facade.imp; + +import java.util.List; + +import javax.ejb.Stateless; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; + +import org.hibernate.Criteria; +import org.hibernate.HibernateException; +import org.hibernate.criterion.Criterion; +import org.hibernate.criterion.Restrictions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import qa.qcri.aidr.analysis.entity.ConfidenceData; +import qa.qcri.aidr.analysis.entity.ConfidenceDataPK; +import qa.qcri.aidr.analysis.facade.ConfidenceStatisticsResourceFacade; +import qa.qcri.aidr.analysis.utils.CommonOperations; +import qa.qcri.aidr.common.logging.ErrorLog; +import qa.qcri.aidr.common.values.ReturnCode; + + +@Stateless +public class ConfidenceStatisticsResourceFacadeImp extends CommonOperations implements ConfidenceStatisticsResourceFacade { + + private static Logger logger = LoggerFactory.getLogger(ConfidenceStatisticsResourceFacadeImp.class); + private static ErrorLog elog = new ErrorLog(); + + @PersistenceContext(unitName = "qa.qcri.aidr.analysis-EJBS") + private EntityManager em; + + @Override + public ReturnCode writeData(ConfidenceData confData) { + try { + em.persist(confData); + //System.out.println("Success in persisting data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + // + ", " + confData.getLabelCode() + ", " + confData.getTimestamp() + ", " + confData.getGranularity() + // + ", " + confData.getBin() + ": " + confData.getCount()); + return ReturnCode.SUCCESS; + } catch (Exception e) { + System.out.println("Failure in persisting data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + + ", " + confData.getLabelCode() + ", " + confData.getTimestamp() + ", " + confData.getGranularity() + + ", " + confData.getBin() + ": " + confData.getCount()); + e.printStackTrace(); + logger.error(elog.toStringException(e)); + return ReturnCode.FAIL; + } + } + + @Override + public ConfidenceData getSingleDataByPK(ConfidenceDataPK confDataPK) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", confDataPK.getCrisisCode())) + .add(Restrictions.eq("timestamp", confDataPK.getTimestamp())) + .add(Restrictions.eq("granularity", confDataPK.getGranularity())) + .add(Restrictions.eq("attributeCode", confDataPK.getAttributeCode())) + .add(Restrictions.eq("labelCode", confDataPK.getLabelCode())) + .add(Restrictions.eq("bin", confDataPK.getBin())); + criteria.add(criterion); + try { + ConfidenceData obj = (ConfidenceData) criteria.uniqueResult(); + return obj; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataByCrisis(String crisisCode) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + criteria.add(Restrictions.eq("crisisCode", crisisCode)); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataByCrisisAttributeLabel(String crisisCode, String attributeCode, String labelCode) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)); + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataByCrisisAttributeLabelGranularity(String crisisCode, String attributeCode, String labelCode, Long granularity) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.eq("granularity", granularity)); + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataAfterTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("timestamp", timestamp)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("timestamp", timestamp)) + .add(Restrictions.eq("granularity", granularity)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataBeforeTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.le("timestamp", timestamp)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataBeforeTimestampGranularity(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.le("timestamp", timestamp)) + .add(Restrictions.eq("granularity", granularity)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataInInterval(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("timestamp", timestamp1)) + .add(Restrictions.le("timestamp", timestamp2)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, + Long timestamp1, Long timestamp2, Long granularity) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.ge("timestamp", timestamp1)) + .add(Restrictions.le("timestamp", timestamp2)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + + //////////////////////////////////////////////// + // Bin related + /////////////////////////////////////////////// + + @Override + public List getDataByCrisisWithBin(String crisisCode, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.ge("bin", bin)); + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataByCrisisAttributeLabelWithBin(String crisisCode, String attributeCode, String labelCode, + Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("bin", bin)); + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataByCrisisAttributeLabelGranularityWithBin(String crisisCode, String attributeCode, String labelCode, + Long granularity, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.ge("bin", bin)); + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataAfterTimestampWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("timestamp", timestamp)) + .add(Restrictions.ge("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataAfterTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("timestamp", timestamp)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.ge("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataBeforeTimestampWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.le("timestamp", timestamp)) + .add(Restrictions.ge("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataBeforeTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.le("timestamp", timestamp)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.ge("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataInIntervalWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp1, + Long timestamp2, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("timestamp", timestamp1)) + .add(Restrictions.le("timestamp", timestamp2)) + .add(Restrictions.ge("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataInIntervalWithGranularityWithBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp1, Long timestamp2, Long granularity, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.ge("timestamp", timestamp1)) + .add(Restrictions.le("timestamp", timestamp2)) + .add(Restrictions.ge("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataByCrisisInBin(String crisisCode, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("bin", bin)); + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataByCrisisAttributeLabelInBin(String crisisCode, String attributeCode, String labelCode, + Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.eq("bin", bin)); + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataByCrisisAttributeLabelGranularityInBin(String crisisCode, String attributeCode, String labelCode, + Long granularity, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.eq("bin", bin)); + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataAfterTimestampInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("timestamp", timestamp)) + .add(Restrictions.eq("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataAfterTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("timestamp", timestamp)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.eq("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataBeforeTimestampInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.le("timestamp", timestamp)) + .add(Restrictions.eq("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataBeforeTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.le("timestamp", timestamp)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.eq("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataInIntervalInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp1, + Long timestamp2, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.ge("timestamp", timestamp1)) + .add(Restrictions.le("timestamp", timestamp2)) + .add(Restrictions.eq("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } + + @Override + public List getDataInIntervalWithGranularityInBin(String crisisCode, String attributeCode, String labelCode, + Long timestamp1, Long timestamp2, Long granularity, Integer bin) { + Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.ge("timestamp", timestamp1)) + .add(Restrictions.le("timestamp", timestamp2)) + .add(Restrictions.eq("bin", bin)); + + criteria.add(criterion); + try { + List objList = (List) criteria.list(); + return objList; + } catch (HibernateException e) { + logger.error(elog.toStringException(e)); + } + return null; + } +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java deleted file mode 100644 index 917fcd2b7..000000000 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/FrequencyStatisticsResourceFacadeImp.java +++ /dev/null @@ -1,616 +0,0 @@ -package qa.qcri.aidr.analysis.facade.imp; - -import java.util.List; - -import javax.ejb.Stateless; -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; - -import org.hibernate.Criteria; -import org.hibernate.HibernateException; -import org.hibernate.criterion.Criterion; -import org.hibernate.criterion.Restrictions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import qa.qcri.aidr.analysis.entity.FrequencyData; -import qa.qcri.aidr.analysis.entity.FrequencyDataPK; -import qa.qcri.aidr.analysis.facade.FrequencyStatisticsResourceFacade; -import qa.qcri.aidr.analysis.utils.CommonOperations; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.common.values.ReturnCode; - -@Stateless -public class FrequencyStatisticsResourceFacadeImp extends CommonOperations implements FrequencyStatisticsResourceFacade { - - private static Logger logger = LoggerFactory.getLogger(FrequencyStatisticsResourceFacadeImp.class); - private static ErrorLog elog = new ErrorLog(); - - @PersistenceContext(unitName = "qa.qcri.aidr.analysis-EJBS") - private EntityManager em; - - @Override - public ReturnCode writeData(FrequencyData freqData) { - try { - em.persist(freqData); - //System.out.println("Success in persisting data for: " + freqData.getCrisisCode() + ", " + freqData.getAttributeCode() - // + ", " + freqData.getLabelCode() + ", " + freqData.getTimestamp() + ", " + freqData.getGranularity() - // + ", " + freqData.getBin() + ": " + freqData.getCount()); - return ReturnCode.SUCCESS; - } catch (Exception e) { - System.out.println("Failure in persisting data for: " + freqData.getCrisisCode() + ", " + freqData.getAttributeCode() - + ", " + freqData.getLabelCode() + ", " + freqData.getTimestamp() + ", " + freqData.getGranularity() - + ", " + freqData.getBin() + ": " + freqData.getCount()); - e.printStackTrace(); - logger.error(elog.toStringException(e)); - return ReturnCode.FAIL; - } - } - - @Override - public FrequencyData getSingleDataByPK(FrequencyDataPK freqDataPK) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", freqDataPK.getCrisisCode())) - .add(Restrictions.eq("timestamp", freqDataPK.getTimestamp())) - .add(Restrictions.eq("granularity", freqDataPK.getGranularity())) - .add(Restrictions.eq("attribute_code", freqDataPK.getAttributeCode())) - .add(Restrictions.eq("label_code", freqDataPK.getLabelCode())) - .add(Restrictions.eq("bin", freqDataPK.getBin())); - criteria.add(criterion); - try { - FrequencyData obj = (FrequencyData) criteria.uniqueResult(); - return obj; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataByCrisis(String crisisCode) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - criteria.add(Restrictions.eq("crisis_code", crisisCode)); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataByCrisisAttributeLabel(String crisisCode, String attributeCode, String labelCode) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)); - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataByCrisisAttributeLabelGranularity(String crisisCode, String attributeCode, String labelCode, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.eq("granularity", granularity)); - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataAfterTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("timestamp", timestamp)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("timestamp", timestamp)) - .add(Restrictions.eq("granularity", granularity)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataBeforeTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.le("timestamp", timestamp)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataBeforeTimestampGranularity(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.le("timestamp", timestamp)) - .add(Restrictions.eq("granularity", granularity)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataInInterval(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("timestamp", timestamp1)) - .add(Restrictions.le("timestamp", timestamp2)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.ge("timestamp", timestamp1)) - .add(Restrictions.le("timestamp", timestamp2)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - - //////////////////////////////////////////////// - // Bin related - /////////////////////////////////////////////// - - @Override - public List getDataByCrisisWithBin(String crisisCode, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.ge("bin", bin)); - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataByCrisisAttributeLabelWithBin(String crisisCode, String attributeCode, String labelCode, - Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("bin", bin)); - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataByCrisisAttributeLabelGranularityWithBin(String crisisCode, String attributeCode, String labelCode, - Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.ge("bin", bin)); - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataAfterTimestampWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("timestamp", timestamp)) - .add(Restrictions.ge("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataAfterTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("timestamp", timestamp)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.ge("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataBeforeTimestampWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.le("timestamp", timestamp)) - .add(Restrictions.ge("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataBeforeTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.le("timestamp", timestamp)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.ge("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataInIntervalWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp1, - Long timestamp2, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("timestamp", timestamp1)) - .add(Restrictions.le("timestamp", timestamp2)) - .add(Restrictions.ge("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataInIntervalWithGranularityWithBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.ge("timestamp", timestamp1)) - .add(Restrictions.le("timestamp", timestamp2)) - .add(Restrictions.ge("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataByCrisisInBin(String crisisCode, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("bin", bin)); - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataByCrisisAttributeLabelInBin(String crisisCode, String attributeCode, String labelCode, - Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.eq("bin", bin)); - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataByCrisisAttributeLabelGranularityInBin(String crisisCode, String attributeCode, String labelCode, - Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.eq("bin", bin)); - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataAfterTimestampInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("timestamp", timestamp)) - .add(Restrictions.eq("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataAfterTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("timestamp", timestamp)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.eq("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataBeforeTimestampInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.le("timestamp", timestamp)) - .add(Restrictions.eq("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataBeforeTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.le("timestamp", timestamp)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.eq("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataInIntervalInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp1, - Long timestamp2, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.ge("timestamp", timestamp1)) - .add(Restrictions.le("timestamp", timestamp2)) - .add(Restrictions.eq("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } - - @Override - public List getDataInIntervalWithGranularityInBin(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) - .add(Restrictions.eq("granularity", granularity)) - .add(Restrictions.ge("timestamp", timestamp1)) - .add(Restrictions.le("timestamp", timestamp2)) - .add(Restrictions.eq("bin", bin)); - - criteria.add(criterion); - try { - List objList = (List) criteria.list(); - return objList; - } catch (HibernateException e) { - logger.error(elog.toStringException(e)); - } - return null; - } -} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java index b4177697c..a718a2965 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java @@ -10,8 +10,6 @@ import org.hibernate.HibernateException; import org.hibernate.criterion.Criterion; import org.hibernate.criterion.Restrictions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import qa.qcri.aidr.analysis.entity.TagData; import qa.qcri.aidr.analysis.entity.TagDataPK; @@ -20,6 +18,7 @@ import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.values.ReturnCode; + @Stateless public class TagDataStatisticsResourceFacadeImp extends CommonOperations implements TagDataStatisticsResourceFacade { @@ -46,94 +45,135 @@ public ReturnCode writeData(TagData tagData) { @Override public TagData getSingleDataByPK(TagDataPK tagDataPK) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", tagDataPK.getCrisisCode())) + .add(Restrictions.eq("crisisCode", tagDataPK.getCrisisCode())) .add(Restrictions.eq("timestamp", tagDataPK.getTimestamp())) .add(Restrictions.eq("granularity", tagDataPK.getGranularity())) - .add(Restrictions.eq("attribute_code", tagDataPK.getAttributeCode())) - .add(Restrictions.eq("label_code", tagDataPK.getLabelCode())); + .add(Restrictions.eq("attributeCode", tagDataPK.getAttributeCode())) + .add(Restrictions.eq("labelCode", tagDataPK.getLabelCode())); criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { TagData obj = (TagData) criteria.uniqueResult(); return obj; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } @Override public List getDataByCrisis(String crisisCode) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - criteria.add(Restrictions.eq("crisis_code", crisisCode)); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + criteria.add(Restrictions.eq("crisisCode", crisisCode)); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } @Override public List getDataByCrisisAttributeLabel(String crisisCode, String attributeCode, String labelCode) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)); + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)); if (labelCode != null) { criterion = Restrictions.conjunction() .add(criterion) - .add(Restrictions.eq("label_code", labelCode)); + .add(Restrictions.eq("labelCode", labelCode)); } criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } @Override public List getDataByCrisisAttributeLabelGranularity(String crisisCode, String attributeCode, String labelCode, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) - .add(Restrictions.eq("label_code", labelCode)) + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("labelCode", labelCode)) .add(Restrictions.eq("granularity", granularity)); criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); + try { + List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); + return objList; + } catch (HibernateException e) { + //logger.error(elog.toStringException(e)); + e.printStackTrace(); + } + return null; + } + + public List getDataByGranularityInTimeWindow(String crisisCode, String attributeCode, String labelCode, + Long timestamp, Long granularity) { + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) + .add(Restrictions.eq("granularity", granularity)) + .add(Restrictions.eq("timestamp", timestamp)); + if (labelCode != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("labelCode", labelCode)); + } + + criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } @Override public List getDataAfterTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) .add(Restrictions.ge("timestamp", timestamp)); if (labelCode != null) { criterion = Restrictions.conjunction() .add(criterion) - .add(Restrictions.eq("label_code", labelCode)); + .add(Restrictions.eq("labelCode", labelCode)); } - criteria.add(criterion); + criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } @@ -141,46 +181,52 @@ public List getDataAfterTimestamp(String crisisCode, String attributeCo @Override public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) .add(Restrictions.ge("timestamp", timestamp)) .add(Restrictions.eq("granularity", granularity)); if (labelCode != null) { criterion = Restrictions.conjunction() .add(criterion) - .add(Restrictions.eq("label_code", labelCode)); + .add(Restrictions.eq("labelCode", labelCode)); } - criteria.add(criterion); + criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } @Override public List getDataBeforeTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) .add(Restrictions.le("timestamp", timestamp)); if (labelCode != null) { criterion = Restrictions.conjunction() .add(criterion) - .add(Restrictions.eq("label_code", labelCode)); + .add(Restrictions.eq("labelCode", labelCode)); } criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } @@ -188,24 +234,27 @@ public List getDataBeforeTimestamp(String crisisCode, String attributeC @Override public List getDataBeforeTimestampGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) .add(Restrictions.le("timestamp", timestamp)) .add(Restrictions.eq("granularity", granularity)); if (labelCode != null) { criterion = Restrictions.conjunction() .add(criterion) - .add(Restrictions.eq("label_code", labelCode)); + .add(Restrictions.eq("labelCode", labelCode)); } - criteria.add(criterion); + criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } @@ -213,23 +262,26 @@ public List getDataBeforeTimestampGranularity(String crisisCode, String @Override public List getDataInInterval(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() - .add(Restrictions.eq("crisis_code", crisisCode)) - .add(Restrictions.eq("attribute_code", attributeCode)) + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.eq("attributeCode", attributeCode)) .add(Restrictions.ge("timestamp", timestamp1)) .add(Restrictions.le("timestamp", timestamp2)); if (labelCode != null) { criterion = Restrictions.conjunction() .add(criterion) - .add(Restrictions.eq("label_code", labelCode)); + .add(Restrictions.eq("labelCode", labelCode)); } - criteria.add(criterion); + criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } @@ -237,19 +289,19 @@ public List getDataInInterval(String crisisCode, String attributeCode, @Override public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(this.getClass()); - Criterion criterion = Restrictions.eq("crisis_code", crisisCode); + Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criterion criterion = Restrictions.eq("crisisCode", crisisCode); // Now add the optional non-null criteria if (attributeCode != null) { criterion = Restrictions.conjunction() .add(criterion) - .add(Restrictions.eq("attribute_code", attributeCode)); + .add(Restrictions.eq("attributeCode", attributeCode)); } if (labelCode != null) { criterion = Restrictions.conjunction() .add(criterion) - .add(Restrictions.eq("label_code", labelCode)); + .add(Restrictions.eq("labelCode", labelCode)); } if (granularity != null) { criterion = Restrictions.conjunction() @@ -266,12 +318,15 @@ public List getDataInIntervalWithGranularity(String crisisCode, String .add(criterion) .add(Restrictions.le("timestamp", timestamp2)); } - criteria.add(criterion); + criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); + System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { //logger.error(elog.toStringException(e)); + e.printStackTrace(); } return null; } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java similarity index 92% rename from aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java rename to aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java index 93ef8072b..25aba3dae 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/FreqDataMapRecord.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java @@ -5,9 +5,9 @@ import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; -public class FreqDataMapRecord extends MapRecord { +public class ConfDataMapRecord extends MapRecord { - public FreqDataMapRecord(List granularityList) { + public ConfDataMapRecord(List granularityList) { this.count = new ConcurrentHashMap(); for (Long g: granularityList) { this.count.put(g, 0); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java index 92aeeb176..79f6355d6 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java @@ -8,6 +8,7 @@ import qa.qcri.aidr.common.logging.ErrorLog; + public class CommonOperations { private Logger logger = Logger.getLogger(CommonOperations.class); private ErrorLog elog = new ErrorLog(); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java index b92b7f889..4e6ba6e4c 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/GranularityData.java @@ -15,6 +15,7 @@ import qa.qcri.aidr.common.code.DateFormatConfig; import qa.qcri.aidr.common.logging.ErrorLog; + public class GranularityData { // Debugging private static Logger logger = Logger.getLogger(GranularityData.class); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java new file mode 100644 index 000000000..5f40edf4c --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java @@ -0,0 +1,15 @@ +package qa.qcri.aidr.analysis.utils; + +import net.minidev.json.JSONObject; + +public class JsonResponse { + public static JSONObject getNewJsonResponseObject(String crisisCode, String attributeCode, Long granularity) { + JSONObject json = new JSONObject(); + json.put("crisis_code", crisisCode); + json.put("attribute_code", attributeCode); + json.put("granularity", granularity); + + return json; + } + +} diff --git a/aidr-analytics/src/main/resources/META-INF/persistence.xml b/aidr-analytics/src/main/resources/META-INF/persistence.xml index a61208031..1912adb70 100644 --- a/aidr-analytics/src/main/resources/META-INF/persistence.xml +++ b/aidr-analytics/src/main/resources/META-INF/persistence.xml @@ -4,12 +4,12 @@ org.hibernate.jpa.HibernatePersistenceProvider JNDI/aidr_analysis qa.qcri.aidr.analysis.entity.TagData - qa.qcri.aidr.analysis.entity.FrequencyData + qa.qcri.aidr.analysis.entity.ConfidenceData - + @@ -29,7 +29,7 @@ org.hibernate.jpa.HibernatePersistenceProvider qa.qcri.aidr.analysis.entity.TagData - qa.qcri.aidr.analysis.entity.FrequencyData + qa.qcri.aidr.analysis.entity.ConfidenceData diff --git a/aidr-analytics/src/main/webapp/WEB-INF/web.xml b/aidr-analytics/src/main/webapp/WEB-INF/web.xml index bc3943ac9..7674866ae 100644 --- a/aidr-analytics/src/main/webapp/WEB-INF/web.xml +++ b/aidr-analytics/src/main/webapp/WEB-INF/web.xml @@ -25,7 +25,7 @@ qa.qcri.aidr.analysis.api.GetTagDataStatistics - qa.qcri.aidr.analysis.api.GetFrequencyStatistics + qa.qcri.aidr.analysis.api.GetConfidenceStatistics qa.qcri.aidr.analysis.api.WriteStatisticsData From 1fd68e4e7e33c011911e688db80ccd28f76a9b45 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Thu, 25 Sep 2014 18:18:55 +0300 Subject: [PATCH 4/6] modified pom to packaging=jar --- aidr-common/pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aidr-common/pom.xml b/aidr-common/pom.xml index 9e2ea3b1d..39f9d4a58 100644 --- a/aidr-common/pom.xml +++ b/aidr-common/pom.xml @@ -4,7 +4,8 @@ qa.qcri.aidr.common aidr-common 1.0 - + jar + 1.7 1.9.13 From 3b620aa38f72085a3e892001d077fe65f41ef7f5 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Thu, 25 Sep 2014 19:24:45 +0300 Subject: [PATCH 5/6] ResponseWrapper example --- .../aidr/common/code/ResponseWrapper.java | 23 +++++++ .../qa/qcri/aidr/utils/ClassifiedTweet.java | 2 +- .../java/qa/qcri/aidr/utils/ResultStatus.java | 65 +++++++------------ 3 files changed, 49 insertions(+), 41 deletions(-) create mode 100644 aidr-common/src/main/java/qa/qcri/aidr/common/code/ResponseWrapper.java diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/code/ResponseWrapper.java b/aidr-common/src/main/java/qa/qcri/aidr/common/code/ResponseWrapper.java new file mode 100644 index 000000000..9edad9d77 --- /dev/null +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/code/ResponseWrapper.java @@ -0,0 +1,23 @@ +package qa.qcri.aidr.common.code; + +import java.util.HashMap; +import java.util.Map; + +public class ResponseWrapper { + public static Map getUIWrapper(String key1, Object value1, String key2, Object value2) { + Map resultMap = new HashMap(2); + resultMap.put(key1, value1); + resultMap.put(key2, value2); + return resultMap; + } + + public static Map getUIWrapper(String collectionCode, String message, String fileName, Boolean success) { + Map modelMap = new HashMap(4); + modelMap.put("code", collectionCode); + modelMap.put("message", message); + modelMap.put("url", fileName); + modelMap.put("success", success); + + return modelMap; + } +} diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java index 8636c6912..3d5ef05c4 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java @@ -16,8 +16,8 @@ import org.apache.log4j.Logger; +import qa.qcri.aidr.common.code.DateFormatConfig; import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.common.values.DateFormatConfig; import qa.qcri.aidr.persister.filter.NominalLabel; /** diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ResultStatus.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ResultStatus.java index c147825ac..48da76cf6 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ResultStatus.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ResultStatus.java @@ -12,56 +12,41 @@ import org.apache.log4j.Logger; import org.glassfish.jersey.jackson.JacksonFeature; +import qa.qcri.aidr.common.code.ResponseWrapper; import qa.qcri.aidr.logging.ErrorLog; -public class ResultStatus { - +public class ResultStatus extends ResponseWrapper { + private static Logger logger = Logger.getLogger(ResultStatus.class); private static ErrorLog elog = new ErrorLog(); - + @Deprecated @SuppressWarnings({ "unused", "unchecked" }) public static Integer getTotalDownloadedCount(final String collectionCode) { - Response clientResponse = null; - Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build(); - try { - WebTarget webResource = client.target(Config.managerUrl - + "/public/collection/findTotalCount?channelCode=" + collectionCode); + Response clientResponse = null; + Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build(); + try { + WebTarget webResource = client.target(Config.managerUrl + + "/public/collection/findTotalCount?channelCode=" + collectionCode); - clientResponse = webResource.request(MediaType.APPLICATION_JSON).get(); - Map collectionMap = new HashMap(); + clientResponse = webResource.request(MediaType.APPLICATION_JSON).get(); + Map collectionMap = new HashMap(); - if (clientResponse.getStatus() == 200) { - //convert JSON string to Map - collectionMap = clientResponse.readEntity(Map.class); - logger.info("Channel info received from manager: " + collectionMap); - if (collectionMap != null) { - return collectionMap.get(collectionCode); - } - } else { - logger.warn("Couldn't contact AIDRFetchManager for publiclyListed status, channel: " + collectionCode); + if (clientResponse.getStatus() == 200) { + //convert JSON string to Map + collectionMap = clientResponse.readEntity(Map.class); + logger.info("Channel info received from manager: " + collectionMap); + if (collectionMap != null) { + return collectionMap.get(collectionCode); } - } catch (Exception e) { - logger.error("Error in querying manager for running collections: " + clientResponse); - logger.error(elog.toStringException(e)); + } else { + logger.warn("Couldn't contact AIDRFetchManager for publiclyListed status, channel: " + collectionCode); } - return null; - } - - public static Map getUIWrapper(String key1, Object value1, String key2, Object value2) { - Map resultMap = new HashMap(2); - resultMap.put(key1, value1); - resultMap.put(key2, value2); - return resultMap; - } - - public static Map getUIWrapper(String collectionCode, String message, String fileName, Boolean success) { - Map modelMap = new HashMap(4); - modelMap.put("code", collectionCode); - modelMap.put("message", message); - modelMap.put("url", fileName); - modelMap.put("success", success); - - return modelMap; + } catch (Exception e) { + logger.error("Error in querying manager for running collections: " + clientResponse); + logger.error(elog.toStringException(e)); + } + return null; } + } From 06b786e23e6b471ae244dde17261749e877cfe06 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Thu, 25 Sep 2014 19:45:38 +0300 Subject: [PATCH 6/6] DB index creation script --- aidr-analytics/db_scripts/update_db.sql | 7 +++++++ aidr-analytics/src/main/resources/granularity.properties | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 aidr-analytics/db_scripts/update_db.sql diff --git a/aidr-analytics/db_scripts/update_db.sql b/aidr-analytics/db_scripts/update_db.sql new file mode 100644 index 000000000..822ebae88 --- /dev/null +++ b/aidr-analytics/db_scripts/update_db.sql @@ -0,0 +1,7 @@ +CREATE INDEX TAG_FETCH_INDEX1 ON aidr_analysis.tag_data (crisis_code, attribute_code, timestamp, granularity); +CREATE INDEX TAG_FETCH_INDEX2 ON aidr_analysis.tag_data (crisis_code, attribute_code); +CREATE INDEX TAG_FETCH_INDEX3 ON aidr_analysis.tag_data (crisis_code); +CREATE INDEX TAG_FETCH_INDEX4 ON aidr_analysis.tag_data (attribute_code); +CREATE INDEX TAG_FETCH_INDEX5 ON aidr_analysis.tag_data (timestamp); +CREATE INDEX TAG_FETCH_INDEX6 ON aidr_analysis.tag_data (crisis_code, attribute_code, label_code, timestamp, granularity); +CREATE INDEX TAG_FETCH_INDEX7 ON aidr_analysis.tag_data (crisis_code, attribute_code, label_code); diff --git a/aidr-analytics/src/main/resources/granularity.properties b/aidr-analytics/src/main/resources/granularity.properties index f3231103c..8fe24e3d3 100644 --- a/aidr-analytics/src/main/resources/granularity.properties +++ b/aidr-analytics/src/main/resources/granularity.properties @@ -1 +1 @@ -granularity=30s,2m,1h \ No newline at end of file +granularity=5m,1h,1d \ No newline at end of file