From 5df9df6a247e8ae767d83bcc7703f95bf1582ffc Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Tue, 21 Jul 2015 19:25:30 +0530 Subject: [PATCH 01/44] Fixed bug in the story #89696224 for downloading human tagged tweets and some minor changes --- .../filter/ClassifiedFilteredTweet.java | 12 ++++---- .../qa/qcri/aidr/dbmanager/dto/CrisisDTO.java | 6 +++- .../dto/HumanLabeledDocumentDTO.java | 6 ++-- .../dto/HumanLabeledDocumentList.java | 2 +- .../controller/TrainingDataController.js | 8 ++--- .../output/utils/ClassifiedTweetAdapter.java | 26 +++++++++++++++++ .../qa/qcri/aidr/utils/ClassifiedTweet.java | 29 +++++-------------- .../qa/qcri/aidr/utils/JsonDeserializer.java | 8 ++--- .../qcri/aidr/predictui/api/MiscResource.java | 4 +++ ...rceImp.java => MiscResourceFacadeImp.java} | 6 ++-- 10 files changed, 62 insertions(+), 45 deletions(-) rename aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/{MiscResourceImp.java => MiscResourceFacadeImp.java} (97%) diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java index 98c41336b..9a6e0651b 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java @@ -30,7 +30,7 @@ public class ClassifiedFilteredTweet implements Serializable { private String text = null; private String crisis_code = null; private String crisis_name = null; - private List nominal_labels; + public List nominal_labels; private String id = null; private String screen_name = null; @@ -210,16 +210,14 @@ public void setCrisisName(String crisisName) { } public List getNominalLabels() { - List arr = new ArrayList(); - if (nominal_labels != null) - arr.addAll(nominal_labels); - return arr; + return nominal_labels!=null ? nominal_labels : new ArrayList(); } public void setNominalLabels(List nLabels) { - if (nominal_labels != null) { - nominal_labels.addAll(nLabels); + if (nominal_labels == null) { + nominal_labels = new ArrayList(); } + nominal_labels.addAll(nLabels); } public String getId() { diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisDTO.java index 03bc59e64..385ec50a7 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisDTO.java @@ -32,7 +32,11 @@ @JsonIgnoreProperties(ignoreUnknown=true) public class CrisisDTO implements Serializable { - private static final long serialVersionUID = -4168326869582663472L; + + /** + * + */ + private static final long serialVersionUID = -7825767671101319130L; static final Logger logger = Logger.getLogger("db-manager-log"); diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java index eea914227..b6fbd56a6 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java @@ -21,12 +21,12 @@ @JsonIgnoreProperties(ignoreUnknown=true) @XmlRootElement public class HumanLabeledDocumentDTO implements Serializable { - + /** * */ - private static final long serialVersionUID = 1L; - + private static final long serialVersionUID = -8284975892012345464L; + @XmlElement private DocumentDTO doc; @XmlElement private List labelData; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java index 5a5fbbfa6..219c82d1e 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java @@ -18,7 +18,7 @@ public class HumanLabeledDocumentList implements Serializable { /** * */ - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 4864042221879747333L; @XmlElement private List items; diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js b/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js index e1c45e896..83171e850 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js @@ -196,12 +196,8 @@ Ext.define('TAGGUI.training-data.controller.TrainingDataController', { queryString: me.mainComponent.constraintsString }; - if(format == 'csv'){ - url = '/protected/tagger/downloadHumanLabeledDocuments.action'; - } else { - params.jsonType = format; - url = '/protected/tagger/downloadHumanLabeledDocuments.action'; - } + params.fileType = format; + url = '/protected/tagger/downloadHumanLabeledDocuments.action'; btn.setDisabled(true); me.mainComponent.downloadLink.setText('
', false); diff --git a/aidr-output/src/test/java/qa/qcri/aidr/output/utils/ClassifiedTweetAdapter.java b/aidr-output/src/test/java/qa/qcri/aidr/output/utils/ClassifiedTweetAdapter.java index 866e13cef..5fd7210b4 100644 --- a/aidr-output/src/test/java/qa/qcri/aidr/output/utils/ClassifiedTweetAdapter.java +++ b/aidr-output/src/test/java/qa/qcri/aidr/output/utils/ClassifiedTweetAdapter.java @@ -16,7 +16,33 @@ public class ClassifiedTweetAdapter { private String text; private String crisis_code; private String crisis_name; + private String geo; + private String latitude; + public String getGeo() { + return geo; + } + + public String getLatitude() { + return latitude; + } + + public String getLongitude() { + return longitude; + } + + public void setGeo(String geo) { + this.geo = geo; + } + + public void setLatitude(String latitude) { + this.latitude = latitude; + } + + public void setLongitude(String longitude) { + this.longitude = longitude; + } + private String longitude; private String id; private String screen_name; diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java index e21d0cf21..1a5d0bd7b 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java @@ -13,15 +13,19 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.TimeZone; import org.apache.log4j.Logger; +import qa.qcri.aidr.common.code.DateFormatConfig; +import qa.qcri.aidr.common.filter.ClassifiedFilteredTweet; +import qa.qcri.aidr.common.filter.NominalLabel; +import qa.qcri.aidr.dbmanager.dto.DocumentNominalLabelDTO; +import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentDTO; + import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; @@ -29,13 +33,6 @@ import com.google.gson.JsonParser; import com.google.gson.stream.JsonReader; -import qa.qcri.aidr.common.code.DateFormatConfig; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.dbmanager.dto.DocumentNominalLabelDTO; -import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentDTO; -import qa.qcri.aidr.common.filter.ClassifiedFilteredTweet; -import qa.qcri.aidr.common.filter.NominalLabel; - public class ClassifiedTweet extends ClassifiedFilteredTweet implements Document, Serializable { @@ -67,7 +64,6 @@ public class ClassifiedTweet extends ClassifiedFilteredTweet implements Documen private String confidence_1; private String humanLabeled_1; - public List nominal_labels; private AidrObject aidr; public ClassifiedTweet() { @@ -331,14 +327,6 @@ public String setDateString(String timeString) { return timeString; } - public List getNominalLabels() { - return nominal_labels; - } - - public void setNominalLabels(List nLabels) { - this.nominal_labels = nLabels; - } - public long getTimestamp() { return this.timestamp; } @@ -523,11 +511,10 @@ public void createDummyAIDRField(String collectionCode) { this.createDummyNominalLabels(collectionCode); } - public void toClassifiedTweetFromLabeledDoc(HumanLabeledDocumentDTO doc) { + public void toClassifiedTweetFromLabeledDoc(HumanLabeledDocumentDTO doc, String collectionCode) { if (doc != null) { try { - this.toClassifiedTweet(doc.getDoc().getData(), doc.getDoc().getCrisisDTO().getCode()); - this.getAidr().setCrisisName(doc.getDoc().getCrisisDTO().getName()); + this.toClassifiedTweet(doc.getDoc().getData(), collectionCode); // Now fill up the nominal_label field if (doc.getLabelData() != null && !doc.getLabelData().isEmpty()) { diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java index fdd981551..9f528bd94 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java @@ -1829,7 +1829,7 @@ public String generateClassifiedList2CSV_100K_BasedOnTweetCountFiltered(String c for (HumanLabeledDocumentDTO dto: labeledItems.getItems()) { ClassifiedTweet tweet = new ClassifiedTweet(); - tweet.toClassifiedTweetFromLabeledDoc(dto); + tweet.toClassifiedTweetFromLabeledDoc(dto, collectionCode); if (0 == currentSize && runningHeader == null && writer == null) { runningHeader = csv.setClassifiedTweetHeader(ReadWriteCSV.ClassifiedTweetCSVHeader, ReadWriteCSV.FIXED_CLASSIFIED_TWEET_HEADER_SIZE, tweet); writer = csv.writeClassifiedTweetsCSV(runningHeader, tweetsList, collectionCode, fileNameforCSVGen, writer); @@ -1926,7 +1926,7 @@ public Map generateClassifiedList2TweetIdsCSVFiltered(String col for (HumanLabeledDocumentDTO dto: labeledItems.getItems()) { ClassifiedTweet tweet = new ClassifiedTweet(); - tweet.toClassifiedTweetFromLabeledDoc(dto); + tweet.toClassifiedTweetFromLabeledDoc(dto, collectionCode); if (0 == totalCount && runningHeader == null && writer == null) { runningHeader = csv.setClassifiedTweetHeader(ReadWriteCSV.ClassifiedTweetCSVHeader, ReadWriteCSV.FIXED_CLASSIFIED_TWEET_HEADER_SIZE, tweet); writer = csv.writeClassifiedTweetsCSV(runningHeader, tweetsList, collectionCode, fileNameforCSVGen, writer); @@ -2032,7 +2032,7 @@ public String generateClassifiedList2JSON_100K_BasedOnTweetCountFiltered(String boolean isDone = false; for (HumanLabeledDocumentDTO dto: labeledItems.getItems()) { ClassifiedTweet tweet = new ClassifiedTweet(); - tweet.toClassifiedTweetFromLabeledDoc(dto); + tweet.toClassifiedTweetFromLabeledDoc(dto, collectionCode); if (currentSize < exportLimit && currentSize < Integer.parseInt(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.TWEETS_EXPORT_LIMIT_100K) )) { // Apply filter on tweet @@ -2135,7 +2135,7 @@ public Map generateClassifiedList2TweetIdsJSONFiltered(String co break; } ClassifiedTweet tweet = new ClassifiedTweet(); - tweet.toClassifiedTweetFromLabeledDoc(dto); + tweet.toClassifiedTweetFromLabeledDoc(dto, collectionCode); // Apply filter on tweet if (satisfiesFilter(queryList, tweetFilter, tweet)) { diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java index ad1680472..95b1442dd 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java @@ -227,6 +227,10 @@ public Response downloadHumanLabeledDocumentsByCrisisIDUserName(String queryStri try { List dtoList = miscEJB.getHumanLabeledDocumentsByCrisisCode(crisisCode, count); logger.info("REST call will return dto List size = " + (dtoList != null ? dtoList.size() : "null")); + if(count==-1){ + count=null; + } + if (dtoList != null) { ResponseWrapper response = new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)); response.setTotal(dtoList.size()); diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceFacadeImp.java similarity index 97% rename from aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceImp.java rename to aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceFacadeImp.java index 754ab957b..de021fce2 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceFacadeImp.java @@ -29,6 +29,8 @@ import org.apache.log4j.Logger; import org.glassfish.jersey.jackson.JacksonFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -38,7 +40,7 @@ * @author Imran */ @Stateless -public class MiscResourceImp implements MiscResourceFacade { +public class MiscResourceFacadeImp implements MiscResourceFacade { private final String persisterMainUrl = TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.AIDR_PERSISTER_URL); @@ -51,7 +53,7 @@ public class MiscResourceImp implements MiscResourceFacade { @EJB private TaskManagerRemote remoteTaskManager; - private static Logger logger = Logger.getLogger(MiscResourceImp.class); + private static Logger logger = Logger.getLogger(MiscResourceFacadeImp.class); @Override public List getTraningDataByCrisisAndAttribute(long crisisID, From 21ba254b11c2f16adfbd4b331c610aa34920e77a Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Fri, 24 Jul 2015 13:26:48 +0530 Subject: [PATCH 02/44] Added a script for running the testers. Minor changes in email-configurations --- .../collectors/TwitterStatusListener.java | 3 +- .../qa/qcri/aidr/common/util/EmailClient.java | 6 +- .../qcri/aidr/predictui/api/MiscResource.java | 6 +- aidr_testers.sh | 62 +++++++++++++++++++ profiles/dev/config.properties | 10 +-- profiles/prod/config.properties | 10 +-- profiles/staging/config.properties | 10 +-- 7 files changed, 85 insertions(+), 22 deletions(-) create mode 100644 aidr_testers.sh diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/TwitterStatusListener.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/TwitterStatusListener.java index 7f5706036..c12cbb175 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/TwitterStatusListener.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/TwitterStatusListener.java @@ -231,9 +231,10 @@ public void sendErrorMail(String code, String errorMsg) { + "/misc/sendErrorEmail"); Form form = new Form(); + form.param("module", "AIDRCollector"); form.param("code", code); form.param("description", errorMsg); - + clientResponse = webResource.request().post( Entity.entity(form,MediaType.APPLICATION_FORM_URLENCODED),Response.class); if (clientResponse.getStatus() != 200) { diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java b/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java index a5a2fea8a..928b3cb47 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java @@ -22,7 +22,7 @@ public class EmailClient { private static CommonConfigurator configProperties = CommonConfigurator.getInstance(); private static Logger logger = Logger.getLogger(EmailClient.class.getName()); - public static void sendErrorMail(String code, String errorMsg) + public static void sendErrorMail(String module, String code, String errorMsg ) { // Recipient's email IDs. String recipients = configProperties.getProperty(CommonConfigurationProperty.RECIPIENT_EMAIL); @@ -50,13 +50,13 @@ protected PasswordAuthentication getPasswordAuthentication() { MimeMessage message = new MimeMessage(session); // Set Subject: header field - message.setSubject(configProperties.getProperty(CommonConfigurationProperty.MAIL_SUBJECT) + code); + message.setSubject(module); Date d = new Date(System.currentTimeMillis()); String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); // Now set the actual message - message.setText(time+configProperties.getProperty(CommonConfigurationProperty.MAIL_BODY) + errorMsg); + message.setText(time+ module + code +"\n"+ errorMsg); // Set From: header field of the header. message.setFrom(new InternetAddress(sender,"AIDR Admin")); diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java index 95b1442dd..b4e13e7aa 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java @@ -253,10 +253,10 @@ public Response downloadHumanLabeledDocumentsByCrisisIDUserName(String queryStri @Consumes(MediaType.APPLICATION_FORM_URLENCODED) @Produces(MediaType.TEXT_PLAIN) @Path("/sendErrorEmail") - public Response sendErrorEmail(@FormParam("code") String code, @FormParam("description") String description) throws Exception { + public Response sendErrorEmail(@FormParam("code") String code, @FormParam("module") String module, @FormParam("description") String description) throws Exception { Boolean emailSent = true; try { - EmailClient.sendErrorMail(code,description); + EmailClient.sendErrorMail(module, code, description); } catch (Exception e) { logger.error("Unable to send email"); logger.error(e.getMessage()); @@ -264,7 +264,7 @@ public Response sendErrorEmail(@FormParam("code") String code, @FormParam("descr } try { - systemEventEJB.insertSystemEvent("ERROR", "Collector", code, description, emailSent); + systemEventEJB.insertSystemEvent("ERROR", module, code, description, emailSent); } catch (Exception e) { return Response.serverError().build(); diff --git a/aidr_testers.sh b/aidr_testers.sh new file mode 100644 index 000000000..6594665b9 --- /dev/null +++ b/aidr_testers.sh @@ -0,0 +1,62 @@ +#!/bin/bash +#commands to run this script as a cron job in every 3 hours +#sudo crontab -e +#Add below line in the file opened by above command +#0 */3 * * * root sh /home/hadoop/AIDR/aidr_testers.sh + +PROFILE=dev +AIDR_HOME=E:/AIDR/AIDR +COLLECTION_TASK_FILE=E:/collectionTask.properties +SEND_MAIL_API=http://localhost:8084/AIDRTaggerAPI/rest/misc/sendErrorEmail + +#Collector Tester +cd $AIDR_HOME +cd aidr-collector +mvn -P$PROFILE test -Dtest=CollectorTesterTest -DcollectionTask=$COLLECTION_TASK_FILE -Dtime=10 +result=$? +#result=0(success), 1(fail) +if [[ $result != 0 ]] ; then + #Sending mail + echo CollectorTester Failed + echo Sending Mail + curl --data "module=AIDRCollector&description=Tester for the collector is failed on $PROFILE" $SEND_MAIL_API +fi + +#Persister Tester +cd $AIDR_HOME +cd aidr-persister +mvn -P$PROFILE test -Dtest=PersisterTesterTest +result=$? +#result=0(success), 1(fail) +if [[ $result != 0 ]] ; then + #Sending mail + echo PersisterTester Failed + echo Sending Mail + curl --data "module=AIDRPersister&description=Tester for the persister is failed on $PROFILE" $SEND_MAIL_API +fi + +#Tagger Tester +cd $AIDR_HOME +cd aidr-tagger +mvn -P$PROFILE test -Dtest=TaggerTesterTest -Dnitems-test=100 +result=$? +#result=0(success), 1(fail) +if [[ $result != 0 ]] ; then + #Sending mail + echo TaggerTester Failed + echo Sending Mail + curl --data "module=AIDRTagger&description=Tester for the tagger is failed on $PROFILE" $SEND_MAIL_API +fi + +#Output Tester +cd $AIDR_HOME +cd aidr-output +mvn -P$PROFILE test -Dtest=OutputTesterTest +result=$? +#result=0(success), 1(fail) +if [[ $result != 0 ]] ; then + #Sending mail + echo OutputTester Failed + echo Sending Mail + curl --data "module=AIDROutput&description=Tester for the output is failed on $PROFILE" $SEND_MAIL_API +fi \ No newline at end of file diff --git a/profiles/dev/config.properties b/profiles/dev/config.properties index 5ee3054bd..871dcc04d 100644 --- a/profiles/dev/config.properties +++ b/profiles/dev/config.properties @@ -92,7 +92,7 @@ MODEL_STORAGE_PATH=model #Comma separated list of recipient emails #In case of fatal errors in the system, emails will be sent -RECIPIENT_EMAILS=mesingh@qf.org.qa,jlucas@qf.org.qa,latika.bhurani@metacube.com,kushalkant.goyal@metacube.com,manish.sharma@metacube.com +RECIPIENT_EMAILS= ############################# # OPTIONAL configuration @@ -121,12 +121,12 @@ REMOTE_TASK_MANAGER_JNDI_PREFIX=java:global/AIDRTaskManager/aidr-task-manager-ej REMOTE_DB_MANAGER_JNDI_PREFIX=java:global/AIDRDBManager/aidr-db-manager-ejb-1.0 #Email configuration -SENDER_EMAIL=aidr.qcri@gmail.com -SENDER_PASS=qcrisc2015 +SENDER_EMAIL= +SENDER_PASS= SMTP_HOST=smtp.gmail.com SMTP_PORT=587 -MAIL_SUBJECT=Twitter Exception in collection: -MAIL_BODY= Unable to start the collection within the max retry attempts. Caused by: +MAIL_SUBJECT= +MAIL_BODY= ############################# # FIXED configuration diff --git a/profiles/prod/config.properties b/profiles/prod/config.properties index 0b458718b..835fe575d 100644 --- a/profiles/prod/config.properties +++ b/profiles/prod/config.properties @@ -93,7 +93,7 @@ MODEL_STORAGE_PATH=model #Comma separated list of recipient emails #In case of fatal errors in the system, emails will be sent -RECIPIENT_EMAILS=mesingh@qf.org.qa,jlucas@qf.org.qa,latika.bhurani@metacube.com,kushalkant.goyal@metacube.com,manish.sharma@metacube.com +RECIPIENT_EMAILS= ############################# # OPTIONAL configuration @@ -122,12 +122,12 @@ REMOTE_TASK_MANAGER_JNDI_PREFIX=java:global/AIDRTaskManager/aidr-task-manager-ej REMOTE_DB_MANAGER_JNDI_PREFIX=java:global/AIDRDBManager/aidr-db-manager-ejb-1.0 #Email configuration -SENDER_EMAIL=aidr.qcri@gmail.com -SENDER_PASS=qcrisc2015 +SENDER_EMAIL= +SENDER_PASS= SMTP_HOST=smtp.gmail.com SMTP_PORT=587 -MAIL_SUBJECT=Twitter Exception in collection: -MAIL_BODY= Unable to start the collection within the max retry attempts. Caused by: +MAIL_SUBJECT= +MAIL_BODY= ############################# # FIXED configuration diff --git a/profiles/staging/config.properties b/profiles/staging/config.properties index 8368c3607..043938f7b 100644 --- a/profiles/staging/config.properties +++ b/profiles/staging/config.properties @@ -93,7 +93,7 @@ MODEL_STORAGE_PATH=model #Comma separated list of recipient emails #In case of fatal errors in the system, emails will be sent -RECIPIENT_EMAILS=mesingh@qf.org.qa,jlucas@qf.org.qa,latika.bhurani@metacube.com,kushalkant.goyal@metacube.com,manish.sharma@metacube.com +RECIPIENT_EMAILS= ############################# # OPTIONAL configuration @@ -122,12 +122,12 @@ REMOTE_TASK_MANAGER_JNDI_PREFIX=java:global/AIDRTaskManager/aidr-task-manager-ej REMOTE_DB_MANAGER_JNDI_PREFIX=java:global/AIDRDBManager/aidr-db-manager-ejb-1.0 #Email configuration -SENDER_EMAIL=aidr.qcri@gmail.com -SENDER_PASS=qcrisc2015 +SENDER_EMAIL= +SENDER_PASS= SMTP_HOST=smtp.gmail.com SMTP_PORT=587 -MAIL_SUBJECT=Twitter Exception in collection: -MAIL_BODY= Unable to start the collection within the max retry attempts. Caused by: +MAIL_SUBJECT= +MAIL_BODY= ############################# # FIXED configuration From 88ac8ee9f694855c0ada8f2d62e468123ef77805 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Fri, 24 Jul 2015 18:30:08 +0530 Subject: [PATCH 03/44] Finished #96994368 Check directories when reading the configuration file --- .../DirectoryNotWritableException.java | 18 ++++++++++++++++++ .../qa/qcri/aidr/common/util/EmailClient.java | 2 +- ...PersisterInitializationContextListener.java | 2 +- .../qcri/aidr/utils/PersisterConfigurator.java | 16 +++++++++++++++- .../predict/common/TaggerConfigurator.java | 13 +++++++++++++ 5 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 aidr-common/src/main/java/qa/qcri/aidr/common/exception/DirectoryNotWritableException.java diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/exception/DirectoryNotWritableException.java b/aidr-common/src/main/java/qa/qcri/aidr/common/exception/DirectoryNotWritableException.java new file mode 100644 index 000000000..797ccb4ed --- /dev/null +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/exception/DirectoryNotWritableException.java @@ -0,0 +1,18 @@ +package qa.qcri.aidr.common.exception; + +public class DirectoryNotWritableException extends RuntimeException { + + /** + * + */ + private static final long serialVersionUID = -8924441840552052829L; + + public DirectoryNotWritableException(String propertyName, + String directoryLocation) { + super( + propertyName + + " = " + + directoryLocation + + " is not writable. Please verify if this is a valid writable directory."); + } +} diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java b/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java index 928b3cb47..ace1468a5 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java @@ -56,7 +56,7 @@ protected PasswordAuthentication getPasswordAuthentication() { String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); // Now set the actual message - message.setText(time+ module + code +"\n"+ errorMsg); + message.setText(time+" "+module +" "+ code +"\n"+ errorMsg); // Set From: header field of the header. message.setFrom(new InternetAddress(sender,"AIDR Admin")); diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/init/PersisterInitializationContextListener.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/init/PersisterInitializationContextListener.java index 71a9aa7df..a5d077c09 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/init/PersisterInitializationContextListener.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/init/PersisterInitializationContextListener.java @@ -18,7 +18,7 @@ public class PersisterInitializationContextListener implements @Override public void contextInitialized(ServletContextEvent event) { - // Initializing the configuration properties. + PersisterConfigurator.getInstance(); } @Override diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterConfigurator.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterConfigurator.java index 56030c9f7..5b6095b85 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterConfigurator.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterConfigurator.java @@ -1,8 +1,11 @@ package qa.qcri.aidr.utils; +import java.io.File; + import org.apache.log4j.Logger; import qa.qcri.aidr.common.code.impl.BaseConfigurator; +import qa.qcri.aidr.common.exception.DirectoryNotWritableException; /** * User: yakubenkova.elena@gmail.com Date: 29.09.14 @@ -15,13 +18,24 @@ public class PersisterConfigurator extends BaseConfigurator { private static final PersisterConfigurator instance = new PersisterConfigurator(); - private PersisterConfigurator() { + private PersisterConfigurator() throws DirectoryNotWritableException { LOGGER.info("Instantiating PersisterConfigurator."); this.initProperties(configLoadFileName, PersisterConfigurationProperty.values()); + + this.directoryIsWritable(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH); } public static PersisterConfigurator getInstance() { return instance; } + private void directoryIsWritable(PersisterConfigurationProperty propertyName) throws DirectoryNotWritableException{ + String directoryLocation = this.getProperty(propertyName); + File f = new File(directoryLocation); + if(!f.canWrite()) { + LOGGER.info(propertyName.getName()+ " = " +directoryLocation+ " is not writable. Please verify if this is a valid writable directory."); + throw new DirectoryNotWritableException(propertyName.getName(), directoryLocation); + } + } + } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerConfigurator.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerConfigurator.java index e81402236..b3e7018f9 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerConfigurator.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerConfigurator.java @@ -1,9 +1,12 @@ package qa.qcri.aidr.predict.common; +import java.io.File; + import org.apache.log4j.Logger; import qa.qcri.aidr.common.code.Configurator; import qa.qcri.aidr.common.code.impl.BaseConfigurator; +import qa.qcri.aidr.common.exception.DirectoryNotWritableException; /** * User: yakubenkova.elena@gmail.com Date: 29.09.14 @@ -20,10 +23,20 @@ public class TaggerConfigurator extends BaseConfigurator { private TaggerConfigurator() { LOGGER.info("Instantiating TaggerConfigurator,"); this.initProperties(configLoadFileName, TaggerConfigurationProperty.values()); + + this.directoryIsWritable(TaggerConfigurationProperty.MODEL_STORE_PATH); } public static Configurator getInstance() { return instance; } + private void directoryIsWritable(TaggerConfigurationProperty propertyName) throws DirectoryNotWritableException{ + String directoryLocation = this.getProperty(propertyName); + File f = new File(directoryLocation); + if(!f.canWrite()) { + LOGGER.info(propertyName.getName()+ " = " +directoryLocation+ " is not writable. Please verify if this is a valid writable directory."); + throw new DirectoryNotWritableException(propertyName.getName(), directoryLocation); + } + } } \ No newline at end of file From ae6e2568b429f331c13f07422f135e3f047dbe0a Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Mon, 27 Jul 2015 13:48:03 +0300 Subject: [PATCH 04/44] aidr-analytics debugging: WIP --- .../analysis/api/WriteStatisticsData.java | 30 ++- .../ConfidenceStatisticsResourceFacade.java | 8 +- .../TagDataStatisticsResourceFacade.java | 7 + ...ConfidenceStatisticsResourceFacadeImp.java | 195 ++++++++++++------ .../TagDataStatisticsResourceFacadeImp.java | 79 +++++-- .../main/resources/META-INF/persistence.xml | 1 + .../src/main/resources/granularity.properties | 2 +- 7 files changed, 230 insertions(+), 92 deletions(-) diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java index 0ceafb531..5c5a049fe 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -49,8 +49,7 @@ */ @Path("/save/") -public class WriteStatisticsData extends ChannelBufferManager implements -ServletContextListener { +public class WriteStatisticsData extends ChannelBufferManager implements ServletContextListener { // Debugging private static Logger logger = Logger.getLogger(WriteStatisticsData.class.getSuperclass()); @@ -74,6 +73,8 @@ public class WriteStatisticsData extends ChannelBufferManager implements private long lastTagDataCheckedTime = 0; private long lastConfDataCheckedTime = 0; + private static ChannelBufferManager cbManager = null; // managing buffers for each publishing channel + @Override public void contextInitialized(ServletContextEvent sce) { logger.info("Initializing channel buffer manager"); @@ -85,7 +86,16 @@ public void contextInitialized(ServletContextEvent sce) { AnalyticsConfigurator configurator = AnalyticsConfigurator.getInstance(); granularityList = configurator.getGranularities(); - initiateChannelBufferManager(GetStatistics.CHANNEL_REG_EX); + if (null == cbManager) { + logger.info("Initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); + System.out.println("[contextInitialized] Initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); + //cbManager = new ChannelBufferManager(CHANNEL_REG_EX); + cbManager = new ChannelBufferManager(); + cbManager.initiateChannelBufferManager(GetStatistics.CHANNEL_REG_EX); + logger.info("Done initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); + System.out.println("[contextInitialized] Done initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); + } + cbManager.initiateChannelBufferManager(GetStatistics.CHANNEL_REG_EX); runFlag = true; t = new Thread(new WriterThread()); @@ -95,6 +105,11 @@ public void contextInitialized(ServletContextEvent sce) { executorServicePool.submit(t); } logger.info("Done initializing channel buffer manager"); + System.out.println("Done initializing channel buffer manager"); + System.out.print("Initialized with granularities: "); + for (Long g: granularityList) { + System.out.print(g + "\t"); + } } @@ -103,7 +118,7 @@ public void contextDestroyed(ServletContextEvent sce) { tagDataMap.clear(); confDataMap.clear(); runFlag = false; - close(); + //close(); logger.info("Context destroyed"); } @@ -139,6 +154,8 @@ private ReturnCode writeOutputDataToTagDataDB(Long granularity,Long timestamp) { TagData t = new TagData(key.getCrisisCode(), timestamp, granularity, key.getAttributeCode(), key.getLabelCode(), tCount.getCount(granularity)); logger.info("Will attempt to persist data for: " + t.getCrisisCode() + ", " + t.getAttributeCode() + ", " + t.getLabelCode() + ", " + t.getTimestamp() + ", " + t.getGranularity() + ": " + t.getCount()); + System.out.println("Will attempt to persist data for: " + t.getCrisisCode() + ", " + t.getAttributeCode() + + ", " + t.getLabelCode() + ", " + t.getTimestamp() + ", " + t.getGranularity() + ": " + t.getCount()); tagDataEJB.writeData(t); tCount.resetCount(granularity); // TagDataMapRecord temp = (TagDataMapRecord) tagDataMap.get(key); @@ -201,13 +218,13 @@ private String getBinNumber(float confidence) { @Override public void manageChannelBuffersWrapper(final String subscriptionPattern, final String channelName, final String receivedMessage) { // logger.info("Firing manageChannelBuffer on message from channel: " + channelName); - manageChannelBuffers(subscriptionPattern, channelName, receivedMessage); + cbManager.manageChannelBuffers(subscriptionPattern, channelName, receivedMessage); if (null == channelName) { logger.error("Something terribly wrong! Fatal error in: " + channelName); } else { try { ClassifiedFilteredTweet classifiedTweet = new ClassifiedFilteredTweet().deserialize(receivedMessage); - if (classifiedTweet != null && classifiedTweet.getNominalLabels() != null) { + if (classifiedTweet != null && classifiedTweet.getNominalLabels() != null && !classifiedTweet.getNominalLabels().isEmpty()) { channelMap.putIfAbsent(classifiedTweet.getCrisisCode(), System.currentTimeMillis()); for (NominalLabel nb : classifiedTweet.getNominalLabels()) { @@ -231,6 +248,7 @@ public void manageChannelBuffersWrapper(final String subscriptionPattern, final TagDataMapRecord t = new TagDataMapRecord(granularityList); tagDataMap.put(tagDataKey, t); logger.info("New Tag map entry with key: " + tagDataKey + " value = " + tagDataMap.get(tagDataKey)); + System.out.println("New Tag map entry with key: " + tagDataKey + " value = " + tagDataMap.get(tagDataKey)); } if (confDataMap.containsKey(confDataKey)) { ConfDataMapRecord f = (ConfDataMapRecord) confDataMap.get(confDataKey); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/ConfidenceStatisticsResourceFacade.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/ConfidenceStatisticsResourceFacade.java index 10af31139..20c13dea1 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/ConfidenceStatisticsResourceFacade.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/ConfidenceStatisticsResourceFacade.java @@ -4,10 +4,12 @@ import java.util.List; import javax.ejb.Local; +import javax.persistence.EntityManager; + +import org.hibernate.Session; import qa.qcri.aidr.analysis.entity.ConfidenceData; import qa.qcri.aidr.analysis.entity.ConfidenceDataPK; -import qa.qcri.aidr.analysis.entity.TagData; import qa.qcri.aidr.common.values.ReturnCode; /** * This facade is not used at the moment. @@ -15,6 +17,10 @@ @Local public interface ConfidenceStatisticsResourceFacade { + public EntityManager getEntityManager(); + public int setEntityManager(EntityManager em); + public Session getCurrentSession(); + public ReturnCode writeData(ConfidenceData freqData); public ConfidenceData getSingleDataByPK(ConfidenceDataPK freqDataPK); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java index c8263960e..b741ba1f6 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java @@ -4,6 +4,9 @@ import java.util.List; import javax.ejb.Local; +import javax.persistence.EntityManager; + +import org.hibernate.Session; import qa.qcri.aidr.analysis.entity.TagData; import qa.qcri.aidr.analysis.entity.TagDataPK; @@ -14,6 +17,10 @@ @Local public interface TagDataStatisticsResourceFacade { + public EntityManager getEntityManager(); + public int setEntityManager(EntityManager em); + public Session getCurrentSession(); + public ReturnCode writeData(TagData tagData); public List getDataByCrisis(String crisisCode); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java index 220647f12..a0d2397cb 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java @@ -9,6 +9,7 @@ import org.hibernate.Criteria; import org.hibernate.HibernateException; +import org.hibernate.Session; import org.hibernate.criterion.Criterion; import org.hibernate.criterion.Restrictions; import org.slf4j.Logger; @@ -17,43 +18,79 @@ import qa.qcri.aidr.analysis.entity.ConfidenceData; import qa.qcri.aidr.analysis.entity.ConfidenceDataPK; import qa.qcri.aidr.analysis.facade.ConfidenceStatisticsResourceFacade; -import qa.qcri.aidr.analysis.utils.CommonOperations; -import qa.qcri.aidr.common.logging.ErrorLog; + import qa.qcri.aidr.common.values.ReturnCode; /** * This class is not used at the moment. */ -@Stateless -public class ConfidenceStatisticsResourceFacadeImp extends CommonOperations implements ConfidenceStatisticsResourceFacade { +@Stateless(name = "ConfidenceStatisticsResourceFacadeImp") +public class ConfidenceStatisticsResourceFacadeImp implements ConfidenceStatisticsResourceFacade { private static Logger logger = LoggerFactory.getLogger(ConfidenceStatisticsResourceFacadeImp.class); - private static ErrorLog elog = new ErrorLog(); @PersistenceContext(unitName = "qa.qcri.aidr.analysis-EJBS") private EntityManager em; + @Override + public EntityManager getEntityManager() { + try { + return em; + } catch (Exception e) { + throw new HibernateException("getEntityManager failed"); + } + } + + @Override + public int setEntityManager(EntityManager em) { + try { + if (null == this.em) { + this.em = em; + logger.info("EntityManager set to new value: " + this.em); + return 1; + } else + logger.info("Skipping setter, since EntityManager already initialized to :" + this.em); + return 0; + } catch (Exception e) { + logger.error("EntityManager setting exception : " + em); + logger.error("exception", e); + throw new HibernateException("setEntityManager failed"); + } + } + + @Override + public Session getCurrentSession() { + try { + return em.unwrap(Session.class); + } catch (Exception e) { + logger.error("exception: ", e); + e.printStackTrace(); + throw new HibernateException("getCurrentSession failed"); + } + } + + @Override public ReturnCode writeData(ConfidenceData confData) { try { em.persist(confData); - //System.out.println("Success in persisting data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() - // + ", " + confData.getLabelCode() + ", " + confData.getTimestamp() + ", " + confData.getGranularity() - // + ", " + confData.getBin() + ": " + confData.getCount()); + System.out.println("Success in persisting data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + + ", " + confData.getLabelCode() + ", " + confData.getTimestamp() + ", " + confData.getGranularity() + + ", " + confData.getBin() + ": " + confData.getCount()); return ReturnCode.SUCCESS; } catch (Exception e) { System.out.println("Failure in persisting data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + ", " + confData.getLabelCode() + ", " + confData.getTimestamp() + ", " + confData.getGranularity() + ", " + confData.getBin() + ": " + confData.getCount()); + logger.error("exception", e); e.printStackTrace(); - logger.error(elog.toStringException(e)); return ReturnCode.ERROR; } } @Override public ConfidenceData getSingleDataByPK(ConfidenceDataPK confDataPK) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", confDataPK.getCrisisCode())) .add(Restrictions.eq("timestamp", confDataPK.getTimestamp())) @@ -66,27 +103,29 @@ public ConfidenceData getSingleDataByPK(ConfidenceDataPK confDataPK) { ConfidenceData obj = (ConfidenceData) criteria.uniqueResult(); return obj; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @Override public List getDataByCrisis(String crisisCode) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); criteria.add(Restrictions.eq("crisisCode", crisisCode)); try { List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @Override public List getDataByCrisisAttributeLabel(String crisisCode, String attributeCode, String labelCode) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -96,14 +135,15 @@ public List getDataByCrisisAttributeLabel(String crisisCode, Str List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @Override public List getDataByCrisisAttributeLabelGranularity(String crisisCode, String attributeCode, String labelCode, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -114,14 +154,15 @@ public List getDataByCrisisAttributeLabelGranularity(String cris List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @Override public List getDataAfterTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -133,7 +174,8 @@ public List getDataAfterTimestamp(String crisisCode, String attr List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -141,7 +183,7 @@ public List getDataAfterTimestamp(String crisisCode, String attr @Override public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -154,14 +196,15 @@ public List getDataAfterTimestampGranularity(String crisisCode, List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @Override public List getDataBeforeTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -173,7 +216,8 @@ public List getDataBeforeTimestamp(String crisisCode, String att List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -181,7 +225,7 @@ public List getDataBeforeTimestamp(String crisisCode, String att @Override public List getDataBeforeTimestampGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -194,14 +238,15 @@ public List getDataBeforeTimestampGranularity(String crisisCode, List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @Override public List getDataInInterval(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -214,7 +259,8 @@ public List getDataInInterval(String crisisCode, String attribut List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -222,7 +268,7 @@ public List getDataInInterval(String crisisCode, String attribut @Override public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -236,7 +282,8 @@ public List getDataInIntervalWithGranularity(String crisisCode, List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -248,7 +295,7 @@ public List getDataInIntervalWithGranularity(String crisisCode, @Override public List getDataByCrisisWithBin(String crisisCode, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.ge("bin", bin)); @@ -257,7 +304,8 @@ public List getDataByCrisisWithBin(String crisisCode, Integer bi List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -265,7 +313,7 @@ public List getDataByCrisisWithBin(String crisisCode, Integer bi @Override public List getDataByCrisisAttributeLabelWithBin(String crisisCode, String attributeCode, String labelCode, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -276,7 +324,8 @@ public List getDataByCrisisAttributeLabelWithBin(String crisisCo List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -284,7 +333,7 @@ public List getDataByCrisisAttributeLabelWithBin(String crisisCo @Override public List getDataByCrisisAttributeLabelGranularityWithBin(String crisisCode, String attributeCode, String labelCode, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -296,7 +345,8 @@ public List getDataByCrisisAttributeLabelGranularityWithBin(Stri List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -304,7 +354,7 @@ public List getDataByCrisisAttributeLabelGranularityWithBin(Stri @Override public List getDataAfterTimestampWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -317,7 +367,8 @@ public List getDataAfterTimestampWithBin(String crisisCode, Stri List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -325,7 +376,7 @@ public List getDataAfterTimestampWithBin(String crisisCode, Stri @Override public List getDataAfterTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -339,7 +390,8 @@ public List getDataAfterTimestampGranularityWithBin(String crisi List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -347,7 +399,7 @@ public List getDataAfterTimestampGranularityWithBin(String crisi @Override public List getDataBeforeTimestampWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -360,7 +412,8 @@ public List getDataBeforeTimestampWithBin(String crisisCode, Str List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -368,7 +421,7 @@ public List getDataBeforeTimestampWithBin(String crisisCode, Str @Override public List getDataBeforeTimestampGranularityWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -382,7 +435,8 @@ public List getDataBeforeTimestampGranularityWithBin(String cris List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -390,7 +444,7 @@ public List getDataBeforeTimestampGranularityWithBin(String cris @Override public List getDataInIntervalWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -404,7 +458,8 @@ public List getDataInIntervalWithBin(String crisisCode, String a List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -412,7 +467,7 @@ public List getDataInIntervalWithBin(String crisisCode, String a @Override public List getDataInIntervalWithGranularityWithBin(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -427,14 +482,15 @@ public List getDataInIntervalWithGranularityWithBin(String crisi List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @Override public List getDataByCrisisInBin(String crisisCode, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("bin", bin)); @@ -443,7 +499,8 @@ public List getDataByCrisisInBin(String crisisCode, Integer bin) List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -451,7 +508,7 @@ public List getDataByCrisisInBin(String crisisCode, Integer bin) @Override public List getDataByCrisisAttributeLabelInBin(String crisisCode, String attributeCode, String labelCode, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -462,7 +519,8 @@ public List getDataByCrisisAttributeLabelInBin(String crisisCode List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -470,7 +528,7 @@ public List getDataByCrisisAttributeLabelInBin(String crisisCode @Override public List getDataByCrisisAttributeLabelGranularityInBin(String crisisCode, String attributeCode, String labelCode, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -482,7 +540,8 @@ public List getDataByCrisisAttributeLabelGranularityInBin(String List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -490,7 +549,7 @@ public List getDataByCrisisAttributeLabelGranularityInBin(String @Override public List getDataAfterTimestampInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -503,7 +562,8 @@ public List getDataAfterTimestampInBin(String crisisCode, String List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -511,7 +571,7 @@ public List getDataAfterTimestampInBin(String crisisCode, String @Override public List getDataAfterTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -525,7 +585,8 @@ public List getDataAfterTimestampGranularityInBin(String crisisC List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -533,7 +594,7 @@ public List getDataAfterTimestampGranularityInBin(String crisisC @Override public List getDataBeforeTimestampInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -546,7 +607,8 @@ public List getDataBeforeTimestampInBin(String crisisCode, Strin List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -554,7 +616,7 @@ public List getDataBeforeTimestampInBin(String crisisCode, Strin @Override public List getDataBeforeTimestampGranularityInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -568,7 +630,8 @@ public List getDataBeforeTimestampGranularityInBin(String crisis List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -576,7 +639,7 @@ public List getDataBeforeTimestampGranularityInBin(String crisis @Override public List getDataInIntervalInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -590,7 +653,8 @@ public List getDataInIntervalInBin(String crisisCode, String att List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } @@ -598,7 +662,7 @@ public List getDataInIntervalInBin(String crisisCode, String att @Override public List getDataInIntervalWithGranularityInBin(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2, Long granularity, Integer bin) { - Criteria criteria = getCurrentSession(em).createCriteria(ConfidenceData.class); + Criteria criteria = getCurrentSession().createCriteria(ConfidenceData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -613,7 +677,8 @@ public List getDataInIntervalWithGranularityInBin(String crisisC List objList = (List) criteria.list(); return objList; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("exception", e); + e.printStackTrace(); } return null; } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java index 58d58e21d..13c5b566c 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java @@ -7,35 +7,74 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; +import org.apache.log4j.Logger; import org.hibernate.Criteria; import org.hibernate.HibernateException; +import org.hibernate.Session; import org.hibernate.criterion.Criterion; import org.hibernate.criterion.Restrictions; import qa.qcri.aidr.analysis.entity.TagData; import qa.qcri.aidr.analysis.entity.TagDataPK; import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; -import qa.qcri.aidr.analysis.utils.CommonOperations; -import qa.qcri.aidr.common.logging.ErrorLog; + import qa.qcri.aidr.common.values.ReturnCode; /** * Implements the operations for the interface TagDataStatisticsResourceFacade */ -@Stateless -public class TagDataStatisticsResourceFacadeImp extends CommonOperations implements TagDataStatisticsResourceFacade { +@Stateless(name = "TagDataStatisticsResourceFacadeImp") +public class TagDataStatisticsResourceFacadeImp implements TagDataStatisticsResourceFacade { - //private static Logger logger = LoggerFactory.getLogger(TagDataStatisticsResourceFacadeImp.class); - //private static ErrorLog elog = new ErrorLog(); + private static Logger logger = Logger.getLogger(TagDataStatisticsResourceFacadeImp.class); + @PersistenceContext(unitName = "qa.qcri.aidr.analysis-EJBS") private EntityManager em; + @Override + public EntityManager getEntityManager() { + try { + return em; + } catch (Exception e) { + throw new HibernateException("getEntityManager failed"); + } + } + + @Override + public int setEntityManager(EntityManager em) { + try { + if (null == this.em) { + this.em = em; + logger.info("EntityManager set to new value: " + this.em); + return 1; + } else + logger.info("Skipping setter, since EntityManager already initialized to :" + this.em); + return 0; + } catch (Exception e) { + logger.error("EntityManager setting exception : " + em); + logger.error("exception", e); + throw new HibernateException("setEntityManager failed"); + } + } + + @Override + public Session getCurrentSession() { + try { + return em.unwrap(Session.class); + } catch (Exception e) { + logger.error("exception: ", e); + e.printStackTrace(); + throw new HibernateException("getCurrentSession failed"); + } + } + + @Override public ReturnCode writeData(TagData tagData) { try { em.persist(tagData); - //System.out.println("Success in persisting data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() - // + ", " + tagData.getLabelCode() + ", " + tagData.getTimestamp() + ", " + tagData.getGranularity() + ": " + tagData.getCount()); + System.out.println("Success in persisting data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + + ", " + tagData.getLabelCode() + ", " + tagData.getTimestamp() + ", " + tagData.getGranularity() + ": " + tagData.getCount()); return ReturnCode.SUCCESS; } catch (Exception e) { System.err.println("Failure in persisting data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() @@ -48,7 +87,7 @@ public ReturnCode writeData(TagData tagData) { @Override public TagData getSingleDataByPK(TagDataPK tagDataPK) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", tagDataPK.getCrisisCode())) .add(Restrictions.eq("timestamp", tagDataPK.getTimestamp())) @@ -67,9 +106,10 @@ public TagData getSingleDataByPK(TagDataPK tagDataPK) { return null; } + @SuppressWarnings("unchecked") @Override public List getDataByCrisis(String crisisCode) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); criteria.add(Restrictions.eq("crisisCode", crisisCode)); System.out.println("Formed criteria: " + criteria.toString()); try { @@ -83,9 +123,10 @@ public List getDataByCrisis(String crisisCode) { return null; } + @SuppressWarnings("unchecked") @Override public List getDataByCrisisAttributeLabel(String crisisCode, String attributeCode, String labelCode) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)); @@ -109,7 +150,7 @@ public List getDataByCrisisAttributeLabel(String crisisCode, String att @Override public List getDataByCrisisAttributeLabelGranularity(String crisisCode, String attributeCode, String labelCode, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -130,7 +171,7 @@ public List getDataByCrisisAttributeLabelGranularity(String crisisCode, public List getDataByGranularityInTimeWindow(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -157,7 +198,7 @@ public List getDataByGranularityInTimeWindow(String crisisCode, String @Override public List getDataAfterTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -184,7 +225,7 @@ public List getDataAfterTimestamp(String crisisCode, String attributeCo @Override public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -210,7 +251,7 @@ public List getDataAfterTimestampGranularity(String crisisCode, String @Override public List getDataBeforeTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -237,7 +278,7 @@ public List getDataBeforeTimestamp(String crisisCode, String attributeC @Override public List getDataBeforeTimestampGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -265,7 +306,7 @@ public List getDataBeforeTimestampGranularity(String crisisCode, String @Override public List getDataInInterval(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) .add(Restrictions.eq("attributeCode", attributeCode)) @@ -292,7 +333,7 @@ public List getDataInInterval(String crisisCode, String attributeCode, @Override public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2, Long granularity) { - Criteria criteria = getCurrentSession(em).createCriteria(TagData.class); + Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.eq("crisisCode", crisisCode); // Now add the optional non-null criteria diff --git a/aidr-analytics/src/main/resources/META-INF/persistence.xml b/aidr-analytics/src/main/resources/META-INF/persistence.xml index ae3ae0bf3..8a9fcaa60 100644 --- a/aidr-analytics/src/main/resources/META-INF/persistence.xml +++ b/aidr-analytics/src/main/resources/META-INF/persistence.xml @@ -12,6 +12,7 @@ + diff --git a/aidr-analytics/src/main/resources/granularity.properties b/aidr-analytics/src/main/resources/granularity.properties index 12c79f808..861316a7e 100644 --- a/aidr-analytics/src/main/resources/granularity.properties +++ b/aidr-analytics/src/main/resources/granularity.properties @@ -8,4 +8,4 @@ # Values can be left at their default values ############################# -granularity=5m, 1h, 1d +granularity=10s, 1m, 2m From b867f7f64db9ebf7aa3452239ce54569f6183f50 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Mon, 27 Jul 2015 18:36:10 +0300 Subject: [PATCH 05/44] aidr-analytics bug fix - needs testing; refactored to removed aidr-output dependency --- aidr-analytics/pom.xml | 4 +- .../analysis/api/GetConfidenceStatistics.java | 22 +- .../qcri/aidr/analysis/api/GetStatistics.java | 3 - .../analysis/api/GetTagDataStatistics.java | 2 +- .../analysis/api/WriteStatisticsData.java | 188 +----- ...ConfidenceStatisticsResourceFacadeImp.java | 4 +- .../TagDataStatisticsResourceFacadeImp.java | 6 +- .../aidr/analysis/stat/ConfDataMapRecord.java | 6 + .../aidr/analysis/stat/TagDataMapRecord.java | 8 + .../aidr/analysis/utils/ChannelBuffer.java | 164 ++++++ .../analysis/utils/ChannelBufferManager.java | 543 ++++++++++++++++++ .../aidr/analysis/utils/ConfCounterKey.java | 10 +- .../qcri/aidr/analysis/utils/CounterKey.java | 9 +- .../analysis/utils/JedisConnectionObject.java | 254 ++++++++ .../utils/OutputConfigurationProperty.java | 24 + .../analysis/utils/OutputConfigurator.java | 35 ++ .../src/main/resources/config.properties | 35 ++ .../filter/ClassifiedFilteredTweet.java | 34 +- 18 files changed, 1142 insertions(+), 209 deletions(-) create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBuffer.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurator.java create mode 100644 aidr-analytics/src/main/resources/config.properties diff --git a/aidr-analytics/pom.xml b/aidr-analytics/pom.xml index fe7b06606..bfa66e216 100644 --- a/aidr-analytics/pom.xml +++ b/aidr-analytics/pom.xml @@ -17,12 +17,12 @@ aidr-common 1.0 - + diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetConfidenceStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetConfidenceStatistics.java index 7462f636a..76742deb2 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetConfidenceStatistics.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetConfidenceStatistics.java @@ -20,7 +20,6 @@ import qa.qcri.aidr.analysis.facade.ConfidenceStatisticsResourceFacade; import qa.qcri.aidr.common.code.DateFormatConfig; -import qa.qcri.aidr.output.getdata.ChannelBufferManager; /** @@ -31,7 +30,7 @@ */ @Path("/confData/") -public class GetConfidenceStatistics extends GetStatistics implements ServletContextListener { +public class GetConfidenceStatistics implements ServletContextListener { // Debugging private static Logger logger = Logger.getLogger(GetConfidenceStatistics.class); @@ -64,25 +63,6 @@ public Response ping() { return Response.ok(obj).build(); } - - @GET - @Path("/manage/restart/{passcode}") - @Produces("application/json") - public Response restartAnalyticsService(@PathParam("passcode") String passcode) { - logger.info("Request received"); - if (passcode.equals("sysadmin2013")) { - if (masterCBManager != null) { - masterCBManager.close(); - } - masterCBManager = new ChannelBufferManager(); - masterCBManager.initiateChannelBufferManager(CHANNEL_REG_EX); - logger.info("aidr-output analytics service restarted..."); - final String statusStr = "{\"aidr-output analytics ConfidenceStats service\":\"RESTARTED\"}"; - return Response.ok(statusStr).build(); - } - return Response.ok(new String("{\"password\":\"invalid\"}")).build(); - } - @Override public void contextDestroyed(ServletContextEvent arg0) { // TODO Auto-generated method stub diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetStatistics.java index ad3770f4f..5b33bfd28 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetStatistics.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetStatistics.java @@ -1,6 +1,5 @@ package qa.qcri.aidr.analysis.api; -import qa.qcri.aidr.output.getdata.ChannelBufferManager; public abstract class GetStatistics { @@ -9,6 +8,4 @@ public abstract class GetStatistics { public static final String CHANNEL_REG_EX = "aidr_predict.*"; public static final String CHANNEL_PREFIX_STRING = "aidr_predict."; - public volatile static ChannelBufferManager masterCBManager = null; // managing buffers for each publishing channel - } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java index 94928fde5..b82f4c862 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java @@ -27,7 +27,7 @@ */ @Path("/tagData/") -public class GetTagDataStatistics extends GetStatistics implements ServletContextListener { +public class GetTagDataStatistics implements ServletContextListener { // Debugging private static Logger logger = Logger.getLogger(GetTagDataStatistics.class); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java index 5c5a049fe..11f0f11db 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -2,12 +2,10 @@ import java.util.Date; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import javax.ejb.EJB; @@ -28,17 +26,16 @@ import qa.qcri.aidr.analysis.entity.ConfidenceData; import qa.qcri.aidr.analysis.entity.TagData; import qa.qcri.aidr.analysis.stat.ConfDataMapRecord; -import qa.qcri.aidr.analysis.stat.MapRecord; import qa.qcri.aidr.analysis.stat.TagDataMapRecord; import qa.qcri.aidr.analysis.utils.AnalyticsConfigurator; +import qa.qcri.aidr.analysis.utils.ChannelBufferManager; import qa.qcri.aidr.analysis.utils.ConfCounterKey; import qa.qcri.aidr.analysis.utils.CounterKey; +import qa.qcri.aidr.analysis.utils.OutputConfigurator; import qa.qcri.aidr.analysis.facade.ConfidenceStatisticsResourceFacade; import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; import qa.qcri.aidr.common.values.ReturnCode; -import qa.qcri.aidr.common.filter.ClassifiedFilteredTweet; -import qa.qcri.aidr.common.filter.NominalLabel; -import qa.qcri.aidr.output.getdata.ChannelBufferManager; + /** @@ -49,13 +46,13 @@ */ @Path("/save/") -public class WriteStatisticsData extends ChannelBufferManager implements ServletContextListener { +public class WriteStatisticsData implements ServletContextListener { // Debugging - private static Logger logger = Logger.getLogger(WriteStatisticsData.class.getSuperclass()); + private static Logger logger = Logger.getLogger(WriteStatisticsData.class); private volatile boolean runFlag = false; - + @EJB private TagDataStatisticsResourceFacade tagDataEJB; @@ -66,43 +63,38 @@ public class WriteStatisticsData extends ChannelBufferManager implements Servlet private Thread t = null; //private static final String SENTINEL = "#"; - private static ConcurrentHashMap tagDataMap = null; - private static ConcurrentHashMap confDataMap = null; - private static ConcurrentHashMap channelMap = null; + //private static ConcurrentHashMap tagDataMap = null; + //private static ConcurrentHashMap confDataMap = null; + //private static ConcurrentHashMap channelMap = null; private List granularityList = null; - private long lastTagDataCheckedTime = 0; - private long lastConfDataCheckedTime = 0; + private static ChannelBufferManager cbManager = null; // managing buffers for each publishing channel @Override public void contextInitialized(ServletContextEvent sce) { logger.info("Initializing channel buffer manager"); - - tagDataMap = new ConcurrentHashMap(); - confDataMap = new ConcurrentHashMap(); - channelMap = new ConcurrentHashMap(); - + AnalyticsConfigurator configurator = AnalyticsConfigurator.getInstance(); granularityList = configurator.getGranularities(); - + if (null == cbManager) { - logger.info("Initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); - System.out.println("[contextInitialized] Initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); - //cbManager = new ChannelBufferManager(CHANNEL_REG_EX); cbManager = new ChannelBufferManager(); cbManager.initiateChannelBufferManager(GetStatistics.CHANNEL_REG_EX); logger.info("Done initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); System.out.println("[contextInitialized] Done initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); } - cbManager.initiateChannelBufferManager(GetStatistics.CHANNEL_REG_EX); - + //tagDataMap = ChannelBufferManager.getTagDataMap(); + //confDataMap = ChannelBufferManager.getConfDataMap(); + //channelMap = ChannelBufferManager.getChannelMap(); + runFlag = true; t = new Thread(new WriterThread()); t.setName("AIDR-Analysis DB Writer Thread"); - executorServicePool = getExecutorServicePool(); + executorServicePool = cbManager.getExecutorServicePool(); if (executorServicePool != null) { executorServicePool.submit(t); + System.out.println("Executor thread pool initialized, submitted thread: " + t.getName() + " to pool = " + executorServicePool); } logger.info("Done initializing channel buffer manager"); System.out.println("Done initializing channel buffer manager"); @@ -115,10 +107,8 @@ public void contextInitialized(ServletContextEvent sce) { @Override public void contextDestroyed(ServletContextEvent sce) { - tagDataMap.clear(); - confDataMap.clear(); runFlag = false; - //close(); + //cbManager.close(); logger.info("Context destroyed"); } @@ -126,7 +116,7 @@ public void contextDestroyed(ServletContextEvent sce) { @Path("/tracked/channels") @Produces(MediaType.APPLICATION_JSON) public Response getTrackedChannelsList() { - Set channelList = getActiveChannelsList(); + Set channelList = cbManager.getActiveChannelsList(); JSONObject json = new JSONObject(); json.put("channels", channelList); return Response.ok(json.toJSONString()).build(); @@ -145,17 +135,10 @@ private ReturnCode writeOutputDataToTagDataDB(Long granularity,Long timestamp) { // logger.info("tagDataMap size: " + (tagDataMap != null ? // tagDataMap.size() : "null")); try { - for (CounterKey key : tagDataMap.keySet()) { - // String[] data = key.split(SENTINEL); - // data[0] = channelName - // data[1] = attributeCode - // data[2] = labelCode - TagDataMapRecord tCount = (TagDataMapRecord) tagDataMap.get(key); + for (CounterKey key : cbManager.getTagDataMap().keySet()) { + System.out.println("Will attempt persistence of tag key: " + key.toString()); + TagDataMapRecord tCount = (TagDataMapRecord) ChannelBufferManager.getTagDataMap().get(key); TagData t = new TagData(key.getCrisisCode(), timestamp, granularity, key.getAttributeCode(), key.getLabelCode(), tCount.getCount(granularity)); - logger.info("Will attempt to persist data for: " + t.getCrisisCode() + ", " + t.getAttributeCode() - + ", " + t.getLabelCode() + ", " + t.getTimestamp() + ", " + t.getGranularity() + ": " + t.getCount()); - System.out.println("Will attempt to persist data for: " + t.getCrisisCode() + ", " + t.getAttributeCode() - + ", " + t.getLabelCode() + ", " + t.getTimestamp() + ", " + t.getGranularity() + ": " + t.getCount()); tagDataEJB.writeData(t); tCount.resetCount(granularity); // TagDataMapRecord temp = (TagDataMapRecord) tagDataMap.get(key); @@ -172,19 +155,13 @@ private ReturnCode writeOutputDataToTagDataDB(Long granularity,Long timestamp) { private ReturnCode writeOutputDataToConfDataDB(Long granularity, Long timestamp) { try { - for (CounterKey key : confDataMap.keySet()) { - // String[] data = key.split(SENTINEL); - // data[0] = channelName - // data[1] = attributeCode - // data[2] = labelCode - // data[3] = bin - ConfDataMapRecord fCount = (ConfDataMapRecord) confDataMap.get(key); + for (CounterKey key : cbManager.getConfDataMap().keySet()) { + System.out.println("Will attempt persistence of conf key: " + key.toString()); + ConfDataMapRecord fCount = (ConfDataMapRecord) ChannelBufferManager.getConfDataMap().get(key); ConfidenceData f = new ConfidenceData(key.getCrisisCode(), timestamp, granularity, key.getAttributeCode(), key.getLabelCode(), Integer.parseInt(((ConfCounterKey) key).getBinNumber()), fCount.getCount(granularity)); confDataEJB.writeData(f); fCount.resetCount(granularity); - // ConfDataMapRecord temp = (ConfDataMapRecord) confDataMap.get(key); - // logger.info("After reset, count for key = " + key + " is = " + temp.getCount(granularity)); } } catch (Exception e) { logger.error("Error in writing to ConfidenceDataDB table!", e); @@ -193,113 +170,6 @@ private ReturnCode writeOutputDataToConfDataDB(Long granularity, Long timestamp) return ReturnCode.SUCCESS; } - private String getBinNumber(float confidence) { - int bin = 0; - for (int i = 10; i <= 100; i += 10) { - if (i > (confidence * 100)) { - return Integer.toString(bin); - } - ++bin; - } - return Integer.toString(bin - 1); - } - - /** - * This method is the 'producer' - producing statistics data to be written - * to the respective tables - * - * @param subscriptionPattern - * REDIS pattern that Jedis is subscribed to - * @param channelName - * Name of the channel on which message received from REDIS - * @param receivedMessage - * Received message for the given channelName - */ - @Override - public void manageChannelBuffersWrapper(final String subscriptionPattern, final String channelName, final String receivedMessage) { - // logger.info("Firing manageChannelBuffer on message from channel: " + channelName); - cbManager.manageChannelBuffers(subscriptionPattern, channelName, receivedMessage); - if (null == channelName) { - logger.error("Something terribly wrong! Fatal error in: " + channelName); - } else { - try { - ClassifiedFilteredTweet classifiedTweet = new ClassifiedFilteredTweet().deserialize(receivedMessage); - if (classifiedTweet != null && classifiedTweet.getNominalLabels() != null && !classifiedTweet.getNominalLabels().isEmpty()) { - channelMap.putIfAbsent(classifiedTweet.getCrisisCode(), System.currentTimeMillis()); - - for (NominalLabel nb : classifiedTweet.getNominalLabels()) { - /* - StringBuffer keyPrefix = new StringBuffer() - .append(classifiedTweet.getCrisisCode()) - .append(SENTINEL).append(nb.attribute_code) - .append(SENTINEL).append(nb.label_code); - String tagDataKey = new String(keyPrefix); - String confDataKey = new String(keyPrefix.append( - SENTINEL).append(getBinNumber(nb.confidence))); - */ - CounterKey tagDataKey = new CounterKey(classifiedTweet.getCrisisCode(), nb.attribute_code, nb.label_code); - CounterKey confDataKey = new ConfCounterKey(classifiedTweet.getCrisisCode(), nb.attribute_code, nb.label_code, getBinNumber(nb.confidence)); - - if (tagDataMap.containsKey(tagDataKey)) { - TagDataMapRecord t = (TagDataMapRecord) tagDataMap.get(tagDataKey); - t.incrementAllCounts(); - tagDataMap.put(tagDataKey, t); - } else { - TagDataMapRecord t = new TagDataMapRecord(granularityList); - tagDataMap.put(tagDataKey, t); - logger.info("New Tag map entry with key: " + tagDataKey + " value = " + tagDataMap.get(tagDataKey)); - System.out.println("New Tag map entry with key: " + tagDataKey + " value = " + tagDataMap.get(tagDataKey)); - } - if (confDataMap.containsKey(confDataKey)) { - ConfDataMapRecord f = (ConfDataMapRecord) confDataMap.get(confDataKey); - f.incrementAllCounts(); - confDataMap.put(confDataKey, f); - } else { - ConfDataMapRecord t = new ConfDataMapRecord(granularityList); - confDataMap.put(confDataKey, t); - logger.info("[manageChannelBuffersWrapper] New Conf map entry with key: " + confDataKey + " value = " + confDataMap.get(confDataKey)); - } - } - } - } catch (Exception e) { - logger.error("Exception", e); - } - } - // Periodically check if any channel is down - if so, delete all - // in-memory data for that channel - lastTagDataCheckedTime = periodicInactiveChannelCheck(lastTagDataCheckedTime, tagDataMap); - lastConfDataCheckedTime = periodicInactiveChannelCheck(lastConfDataCheckedTime, confDataMap); - } - - private long periodicInactiveChannelCheck(long lastCheckedTime, ConcurrentHashMap dataMap) { - long currentTime = System.currentTimeMillis(); - if (currentTime - lastCheckedTime > CHECK_INTERVAL) { - for (String key : channelMap.keySet()) { - if ((currentTime - channelMap.get(key)) > NO_DATA_TIMEOUT) { - logger.info("Deleting data for inactive channel = " + key); - int deleteCount = deleteMapRecordsForCollection(key, dataMap); - logger.info("Deleted records count for inactive channel <" + key + "> is = " + deleteCount); - } - } - } - return currentTime; - } - - private int deleteMapRecordsForCollection(final String deleteCollectionCode, ConcurrentHashMap dataMap) { - int count = 0; - Iterator itr = dataMap.keySet().iterator(); - while (itr.hasNext()) { - CounterKey keyVal = itr.next(); - MapRecord data = (MapRecord) dataMap.get(keyVal); - if (keyVal.getCrisisCode().equals(deleteCollectionCode) && data != null && data.isCountZeroForAllGranularity()) { - synchronized (dataMap) { // prevent modification while deletion attempt - dataMap.remove(keyVal); - ++count; - } - } - } - return count; - } private class WriterThread implements Runnable { @@ -310,6 +180,7 @@ private class WriterThread implements Runnable { @Override public void run() { logger.info("Started aidr-analytics writer thread: " + t.getName()); + System.out.println("Started aidr-analytics writer thread: " + t.getName()); Map lastTagDataWriteTime = new TreeMap(); Map lastConfDataWriteTime = new TreeMap(); for (Long g : granularityList) { @@ -327,6 +198,7 @@ public void run() { logger.info("retVal = " + retVal); if (ReturnCode.SUCCESS.equals(retVal)) { logger.info("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); + System.out.println("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); } } if (0 == lastConfDataWriteTime.get(granularity) || (currentTime - lastConfDataWriteTime.get(granularity)) >= granularity) { @@ -345,8 +217,8 @@ public void run() { } } } - //logger.info("Done work - exiting thread: " + t.getName()); } } - + + } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java index a0d2397cb..eac117ce2 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java @@ -74,12 +74,12 @@ public Session getCurrentSession() { public ReturnCode writeData(ConfidenceData confData) { try { em.persist(confData); - System.out.println("Success in persisting data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + System.out.println("Success in persisting conf data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + ", " + confData.getLabelCode() + ", " + confData.getTimestamp() + ", " + confData.getGranularity() + ", " + confData.getBin() + ": " + confData.getCount()); return ReturnCode.SUCCESS; } catch (Exception e) { - System.out.println("Failure in persisting data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + System.out.println("Failure in persisting conf data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + ", " + confData.getLabelCode() + ", " + confData.getTimestamp() + ", " + confData.getGranularity() + ", " + confData.getBin() + ": " + confData.getCount()); logger.error("exception", e); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java index 13c5b566c..f0627ece6 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java @@ -73,14 +73,14 @@ public Session getCurrentSession() { public ReturnCode writeData(TagData tagData) { try { em.persist(tagData); - System.out.println("Success in persisting data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + System.out.println("Success in persisting tag data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + ", " + tagData.getLabelCode() + ", " + tagData.getTimestamp() + ", " + tagData.getGranularity() + ": " + tagData.getCount()); return ReturnCode.SUCCESS; } catch (Exception e) { - System.err.println("Failure in persisting data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + System.out.println("Failure in persisting tag data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + ", " + tagData.getLabelCode() + ", " + tagData.getTimestamp() + ", " + tagData.getGranularity() + ": " + tagData.getCount()); e.printStackTrace(); - //logger.error(elog.toStringException(e)); + logger.error("exception: ", e); return ReturnCode.ERROR; } } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java index 25aba3dae..50c5c585c 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java @@ -8,6 +8,7 @@ public class ConfDataMapRecord extends MapRecord { public ConfDataMapRecord(List granularityList) { + System.out.println("Conf data granularityList: " + granularityList); this.count = new ConcurrentHashMap(); for (Long g: granularityList) { this.count.put(g, 0); @@ -59,4 +60,9 @@ public boolean isCountZeroForAllGranularity() { } return flag; } + + @Override + public String toString() { + return "ConfDataMapRecord [count=" + count + "]"; + } } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java index aa86baf9f..019c010a9 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java @@ -7,8 +7,11 @@ public class TagDataMapRecord extends MapRecord { + + public TagDataMapRecord(List granularityList) { this.count = new ConcurrentHashMap(); + System.out.println("Tag data granulairtyList: " + granularityList); for (Long g: granularityList) { this.count.put(g, 0); } @@ -59,4 +62,9 @@ public boolean isCountZeroForAllGranularity() { } return flag; } + + @Override + public String toString() { + return "TagDataMapRecord [count=" + count + "]"; + } } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBuffer.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBuffer.java new file mode 100644 index 000000000..e2635b6d6 --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBuffer.java @@ -0,0 +1,164 @@ +/** + * @author Koushik Sinha + * Last modified: 06/01/2014 + * + * The ChannelBuffer class implements the creation, + * and deletion of channels as well as retrieval of + * messages from a specific channel. + * + */ + +package qa.qcri.aidr.analysis.utils; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.commons.collections.Buffer; +import org.apache.commons.collections.BufferUtils; +import org.apache.commons.collections.buffer.CircularFifoBuffer; + + +import org.apache.log4j.Logger; + + + +public class ChannelBuffer { + public static int MAX_BUFFER_SIZE = 2000; // number of elements the buffer will hold at any time + public static int MAX_FETCH_SIZE = 2000; // max. number of elements to fetch in one op + private String channelName = null; + private long lastAddTime; + private Buffer messageBuffer = null; + int size = 0; + + private Boolean publiclyListed = true; + + private static Logger logger = Logger.getLogger(ChannelBuffer.class); + + public ChannelBuffer(final String name, final int bufferSize) { + this.channelName = name; + this.messageBuffer = BufferUtils.synchronizedBuffer(new CircularFifoBuffer(bufferSize)); + this.size = bufferSize; + } + + public ChannelBuffer(final String name) { + this.channelName = name; + } + + public void createChannelBuffer() { + this.messageBuffer = BufferUtils.synchronizedBuffer(new CircularFifoBuffer(MAX_BUFFER_SIZE)); + this.size = MAX_BUFFER_SIZE; + } + + public void createChannelBuffer(final int bufferSize) { + this.messageBuffer = BufferUtils.synchronizedBuffer(new CircularFifoBuffer(bufferSize)); + this.size = bufferSize; + } + + public void setPubliclyListed(Boolean publiclyListed) { + this.publiclyListed = publiclyListed; + } + + public Boolean getPubliclyListed() { + return publiclyListed; + } + + @SuppressWarnings("unchecked") + public void addMessage(String msg) { + try { + //synchronized(this.messageBuffer) + { + messageBuffer.add(msg); + lastAddTime = new Date().getTime(); + } + } catch (Exception e) { + logger.error("Couldn't add message to buffer for channel: " + this.channelName); + logger.error("exception", e); + } + } + + @SuppressWarnings("unchecked") + public void addAllMessages(ArrayList msgList) { + try { + //synchronized(this.messageBuffer) + { + messageBuffer.addAll(msgList); + lastAddTime = new Date().getTime(); + } + } catch (Exception e) { + logger.error("Couldn't add message to buffer for channel: " + this.channelName); + logger.error("exception", e); + } + } + + @SuppressWarnings("unchecked") + /** + * @param msgCount: number of messages to return + * @return Returns a list of messages sorted in ascending order of timestamp + */ + public List getMessages(int msgCount) { + //long startTime = System.currentTimeMillis(); + + List tempList = null; + try { + synchronized(this.messageBuffer) { + tempList = new ArrayList(this.messageBuffer.size()); + tempList.addAll(this.messageBuffer); + } + //logger.info("Copied data : " + tempList.size() + ", msgCount:messageBuffer.size = " + msgCount + ":" + messageBuffer.size()); + if (msgCount >= tempList.size()) { + return tempList; // optimization + } + + // Otherwise, get the last msgCount elements, in oldest-first order + // Collections.reverse(tempList); // in-situ reversal O(n) time + List returnList = new ArrayList(msgCount); + int index = Math.max(0, (tempList.size() - msgCount)); // from where to pick + for (int i = index;i < tempList.size();i++) { + returnList.add(tempList.get(i)); + } + //logger.info("Fetched size = " + msgCount + " from start loc = " + Math.max(0, (tempList.size() - msgCount))); + return returnList; + } catch (Exception e) { + logger.error("Error in creating list out of buffered messages"); + logger.error("exception", e); + return null; + } + } + + public int getCurrentMsgCount() { + return messageBuffer.size(); + } + + public void deleteBuffer() { + channelName = null; + if (messageBuffer != null) { + messageBuffer.clear(); + messageBuffer = null; + } + } + + public void setChannelName(String name) { + channelName = name; + } + + public String getChannelName() { + return channelName; + } + + public long getLastAddTime() { + return lastAddTime; + } + + public int getMaxBufferSize() { + return MAX_BUFFER_SIZE; + } + + public int getMaxFetchSize() { + return MAX_FETCH_SIZE; + } + + public int getBufferSize() { + return size; + } +} \ No newline at end of file diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java new file mode 100644 index 000000000..c18a9bc83 --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java @@ -0,0 +1,543 @@ +/** + * @author Koushik Sinha + * Last modified: 06/01/2014 + * + * The ChannelBufferManager class implements the following channel buffer management functionalities + * for the /getLast REST API: + * a) Subscribe to all CHANNEL_PREFIX_STRING channels in REDIS using pattern-based subscription. + * b) Monitor the REDIS pubsub system for new or removed channels. + * c) If new channel found, then create a new ChannelBuffer for it. + * d) If no messages received on an existing channel for a certain duration then delete the channel. + * e) Return an ArrayList of messages for a specific channel and messageCount value. + * + */ + +package qa.qcri.aidr.analysis.utils; + + + +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + + + + + + +import org.apache.log4j.Logger; + + + + + + +import qa.qcri.aidr.analysis.stat.ConfDataMapRecord; +import qa.qcri.aidr.analysis.stat.MapRecord; +import qa.qcri.aidr.analysis.stat.TagDataMapRecord; +import qa.qcri.aidr.common.code.Configurator; +import qa.qcri.aidr.common.filter.ClassifiedFilteredTweet; +import qa.qcri.aidr.common.filter.NominalLabel; +import qa.qcri.aidr.common.redis.LoadShedder; +import qa.qcri.aidr.analysis.utils.OutputConfigurator; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPubSub; +import redis.clients.jedis.exceptions.JedisConnectionException; + +public class ChannelBufferManager { + //private static OutputConfigurator configProperties = OutputConfigurator.getInstance(); + public static final int NO_DATA_TIMEOUT = 48 * 60 * 60 * 1000; // when to delete a channel buffer + public static final int CHECK_INTERVAL = NO_DATA_TIMEOUT; + public static final int CHECK_CHANNEL_PUBLIC_INTERVAL = 5 * 60 * 1000; + + private static int PERSISTER_LOAD_LIMIT; + private static int PERSISTER_LOAD_CHECK_INTERVAL_MINUTES; + + private static Logger logger = Logger.getLogger(ChannelBufferManager.class); + + // Thread related + private static ExecutorService executorServicePool = null; + private volatile static boolean shutdownFlag = false; + + // Redis connection related + public static String redisHost; //= configProperties.getProperty(OutputConfigurationProperty.REDIS_HOST); + public static int redisPort; //= Integer.valueOf(configProperties.getProperty(OutputConfigurationProperty.REDIS_PORT)); + + // Jedis related + public static JedisConnectionObject jedisConn = null; // we need only a single instance of JedisConnectionObject running in background + public static Jedis subscriberJedis = null; + public static RedisSubscriber aidrSubscriber = null; + + // Runtime related + private static boolean isConnected = false; + private static boolean isSubscribed =false; + + private static int bufferSize = -1; + + // Channel Buffering Algorithm related + public static String CHANNEL_PREFIX_STRING; //= configProperties.getProperty(OutputConfigurationProperty.TAGGER_CHANNEL_BASENAME)+"."; + public static Set subscribedChannels; + + // DB access related + //private static DatabaseInterface dbController = null; + private static String managerMainUrl = null; + + private static ConcurrentHashMap redisLoadShedder = null; + + private static ConcurrentHashMap tagDataMap = null; + private static ConcurrentHashMap confDataMap = null; + private static ConcurrentHashMap channelMap = null; + private List granularityList = null; + private long lastTagDataCheckedTime = 0; + private long lastConfDataCheckedTime = 0; + ////////////////////////////////////////// + // ********* Method definitions ********* + ////////////////////////////////////////// + + // Constructor + public ChannelBufferManager() {} + + public void initiateChannelBufferManager(final String channelRegEx) { + Configurator configurator = OutputConfigurator.getInstance(); + redisLoadShedder = new ConcurrentHashMap(); + redisHost = configurator.getProperty(OutputConfigurationProperty.REDIS_HOST); + redisPort = Integer.parseInt(configurator.getProperty(OutputConfigurationProperty.REDIS_PORT)); + CHANNEL_PREFIX_STRING = configurator.getProperty(OutputConfigurationProperty.TAGGER_CHANNEL_BASENAME)+"."; + PERSISTER_LOAD_CHECK_INTERVAL_MINUTES = Integer.parseInt(configurator.getProperty(OutputConfigurationProperty.PERSISTER_LOAD_CHECK_INTERVAL_MINUTES)); + PERSISTER_LOAD_LIMIT = Integer.parseInt(configurator.getProperty(OutputConfigurationProperty.PERSISTER_LOAD_LIMIT)); + + AnalyticsConfigurator analyticsConfigurator = AnalyticsConfigurator.getInstance(); + granularityList = analyticsConfigurator .getGranularities(); + + tagDataMap = new ConcurrentHashMap(); + confDataMap = new ConcurrentHashMap(); + channelMap = new ConcurrentHashMap(); + + managerMainUrl = configurator.getProperty(OutputConfigurationProperty.MANAGER_URL); + logger.info("Initializing channel buffer manager."); + System.out.println("[ChannelBufferManager] Initializing channel buffer manager with values: <" + redisHost + ", " + redisPort + + ", " + PERSISTER_LOAD_CHECK_INTERVAL_MINUTES + ", " + PERSISTER_LOAD_LIMIT + ", " + managerMainUrl + ">"); + + bufferSize = -1; + executorServicePool = Executors.newCachedThreadPool(); //Executors.newFixedThreadPool(10); // max number of threads + logger.info("Created thread pool: " + executorServicePool); + + jedisConn = new JedisConnectionObject(redisHost, redisPort); + try { + subscriberJedis = jedisConn.getJedisResource(); + if (subscriberJedis != null) isConnected = true; + } catch (JedisConnectionException e) { + subscriberJedis = null; + isConnected = false; + logger.error("Fatal error! Couldn't establish connection to REDIS!"); + logger.error("exception", e); + e.printStackTrace(); + } + if (isConnected) { + aidrSubscriber = new RedisSubscriber(); + jedisConn.setJedisSubscription(subscriberJedis, true); // we will be using pattern-based subscription + logger.info("Created new Jedis connection: " + subscriberJedis); + try { + subscribeToChannel(channelRegEx); + isSubscribed = true; + aidrSubscriber.setChannelName(channelRegEx); + logger.info("Created pattern subscription for pattern: " + channelRegEx); + System.out.println("Created pattern subscription for pattern: " + channelRegEx); + } catch (Exception e) { + isSubscribed = false; + logger.error("Fatal exception occurred attempting subscription: " + e.toString()); + logger.error("exception", e); + } + if (isSubscribed) { + subscribedChannels = new HashSet(); + } + } + + } + + public static ConcurrentHashMap getConfDataMap() { + return confDataMap; + } + + public static void setConfDataMap(ConcurrentHashMap confDataMap) { + ChannelBufferManager.confDataMap = confDataMap; + } + + public static ConcurrentHashMap getChannelMap() { + return channelMap; + } + + public static void setChannelMap(ConcurrentHashMap channelMap) { + ChannelBufferManager.channelMap = channelMap; + } + + public static ConcurrentHashMap getTagDataMap() { + return tagDataMap; + } + + public static void setTagDataMap(ConcurrentHashMap tagDataMap) { + ChannelBufferManager.tagDataMap = tagDataMap; + } + + public ExecutorService getExecutorServicePool() { + return executorServicePool; + } + + public void initiateChannelBufferManager(final int bufferSize, final String channelRegEx) { + initiateChannelBufferManager(channelRegEx); // call default constructor + this.bufferSize = bufferSize; // set buffer size + } + + + /** + * This method is the 'producer' - producing statistics data to be written + * to the respective tables + * + * @param subscriptionPattern + * REDIS pattern that Jedis is subscribed to + * @param channelName + * Name of the channel on which message received from REDIS + * @param receivedMessage + * Received message for the given channelName + */ + + public void manageChannelBuffers(final String subscriptionPattern, final String channelName, final String receivedMessage) { + if (null == channelName) { + logger.error("Something terribly wrong! Fatal error in: " + channelName); + } else { + if (!isChannelPresent(channelName)) { + System.out.println("New collection/channel found: " + channelName); + subscribedChannels.add(channelName); + } + try { + ClassifiedFilteredTweet classifiedTweet = new ClassifiedFilteredTweet().deserialize(receivedMessage); + if (classifiedTweet != null && classifiedTweet.getNominalLabels() != null && !classifiedTweet.getNominalLabels().isEmpty()) { + channelMap.putIfAbsent(classifiedTweet.getCrisisCode(), System.currentTimeMillis()); + System.out.println("Found a valid classified tweet for collection: " + classifiedTweet.getCrisisCode() + ", tweet = " + receivedMessage); + for (NominalLabel nb : classifiedTweet.getNominalLabels()) { + if (nb.attribute_code != null && nb.label_code != null) { + CounterKey tagDataKey = new CounterKey(classifiedTweet.getCrisisCode(), nb.attribute_code, nb.label_code); + CounterKey confDataKey = new ConfCounterKey(classifiedTweet.getCrisisCode(), nb.attribute_code, nb.label_code, getBinNumber(nb.confidence)); + + if (tagDataMap.containsKey(tagDataKey)) { + TagDataMapRecord t = (TagDataMapRecord) tagDataMap.get(tagDataKey); + t.incrementAllCounts(); + tagDataMap.put(tagDataKey, t); + System.out.println("Updated Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); + } else { + TagDataMapRecord t = new TagDataMapRecord(granularityList); + tagDataMap.put(tagDataKey, t); + logger.info("New Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); + System.out.println("New Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); + } + if (confDataMap.containsKey(confDataKey)) { + ConfDataMapRecord f = (ConfDataMapRecord) confDataMap.get(confDataKey); + f.incrementAllCounts(); + confDataMap.put(confDataKey, f); + System.out.println("Updated Conf map entry with key: " + confDataKey.toString() + " value = " + confDataMap.get(confDataKey).toString()); + } else { + ConfDataMapRecord t = new ConfDataMapRecord(granularityList); + confDataMap.put(confDataKey, t); + logger.info("[manageChannelBuffersWrapper] New Conf map entry with key: " + confDataKey + " value = " + confDataMap.get(confDataKey)); + System.out.println("New Conf map entry with key: " + confDataKey.toString() + " value = " + confDataMap.get(confDataKey).toString()); + } + } + } + } + } catch (Exception e) { + logger.error("Exception", e); + e.printStackTrace(); + } + } + // Periodically check if any channel is down - if so, delete all + // in-memory data for that channel + lastTagDataCheckedTime = periodicInactiveChannelCheck(lastTagDataCheckedTime, tagDataMap); + lastConfDataCheckedTime = periodicInactiveChannelCheck(lastConfDataCheckedTime, confDataMap); + } + + private long periodicInactiveChannelCheck(long lastCheckedTime, ConcurrentHashMap dataMap) { + long currentTime = System.currentTimeMillis(); + if (currentTime - lastCheckedTime > ChannelBufferManager.CHECK_INTERVAL) { + for (String key : channelMap.keySet()) { + if ((currentTime - channelMap.get(key)) > ChannelBufferManager.NO_DATA_TIMEOUT) { + logger.info("Deleting data for inactive channel = " + key); + int deleteCount = deleteMapRecordsForCollection(key, dataMap); + logger.info("Deleted records count for inactive channel <" + key + "> is = " + deleteCount); + } + } + } + return currentTime; + } + + private int deleteMapRecordsForCollection(final String deleteCollectionCode, ConcurrentHashMap dataMap) { + int count = 0; + Iterator itr = dataMap.keySet().iterator(); + while (itr.hasNext()) { + CounterKey keyVal = itr.next(); + MapRecord data = (MapRecord) dataMap.get(keyVal); + if (keyVal.getCrisisCode().equals(deleteCollectionCode) && data != null && data.isCountZeroForAllGranularity()) { + synchronized (dataMap) { // prevent modification while deletion attempt + dataMap.remove(keyVal); + ++count; + } + } + } + return count; + } + + // Returns true if channelName present in list of channels + // TODO: define the appropriate collections data structure - HashMap, HashSet, ArrayList? + public boolean isChannelPresent(String channelName) { + try { + //logger.info("Checking channelName: " + channelName + ", result = " + subscribedChannels.containsKey(channelName) + ", with message count = " + subscribedChannels.get(channelName).getCurrentMsgCount()); + return (subscribedChannels != null) ? subscribedChannels.contains(channelName) : false; + } catch (Exception e) { + logger.error(Thread.currentThread().getName() + ":: Unable to check if channel present: " + channelName); + e.printStackTrace(); + return false; + } + } + + + + /** + * @return A set of fully qualified channel names, null if none found + */ + public Set getActiveChannelsList() { + try { + Set channelSet = (subscribedChannels != null && !subscribedChannels.isEmpty()) ? subscribedChannels : null; + return channelSet; + } catch (Exception e) { + logger.error("Unable to fetch list of active channels"); + e.printStackTrace(); + return null; + } + } + + + @SuppressWarnings("unused") + private void subscribeToChannel(final String channelRegEx) throws Exception { + Future redisThread = executorServicePool.submit(new Runnable() { + public void run() { + Thread.currentThread().setName("ChannelBufferManager Redis subscription Thread"); + logger.info("New thread <" + Thread.currentThread().getName() + "> created for subscribing to redis channel: " + channelRegEx); + System.out.println("New thread <" + Thread.currentThread().getName() + "> created for subscribing to redis channel: " + channelRegEx); + try { + // Execute the blocking REDIS subscription call + subscriberJedis.psubscribe(aidrSubscriber, channelRegEx); + } catch (JedisConnectionException e) { + logger.error("AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx); + System.out.println("[subscribeToChannel] AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx); + e.printStackTrace(); + stopSubscription(); + Thread.currentThread().interrupt(); + } + Thread.currentThread().interrupt(); + logger.info("Exiting thread: " + Thread.currentThread().getName()); + } + }); + } + + private void stopSubscription() { + try { + if (aidrSubscriber != null && aidrSubscriber.getSubscribedChannels() > 0) { + logger.info("Stopsubscription attempt for channel: " + aidrSubscriber.getChannelName()); + aidrSubscriber.punsubscribe(); + logger.info("Unsubscribed from channel pattern: " + CHANNEL_PREFIX_STRING); + } + } catch (JedisConnectionException e) { + logger.error("Connection to REDIS seems to be lost!"); + logger.error("exception", e); + } + try { + if (jedisConn != null && aidrSubscriber != null && subscriberJedis != null) { + jedisConn.returnJedis(subscriberJedis); + subscriberJedis = null; + logger.info("Stopsubscription completed..."); + System.out.println("[stopSubscription] Stopsubscription completed..."); + } + } catch (Exception e) { + logger.error("Failed to return Jedis resource"); + logger.error("exception", e); + } + logger.info("isShutDown initiated = " + shutdownFlag); + if (!shutdownFlag) { + attemptResubscription(); + } + + } + + private void attemptResubscription() { + int attempts = 0; + boolean isSetup = false; + final int MAX_RECONNECT_ATTEMPTS = 10; + while (!isSetup && attempts < MAX_RECONNECT_ATTEMPTS && !shutdownFlag) { + try { + Thread.sleep(60000); + logger.info("Attempting to resubscribe to REDIS, with jedisConn = " + jedisConn); + if (jedisConn != null) { + isSetup = setupRedisConnection(CHANNEL_PREFIX_STRING+"*"); + } else { + jedisConn = new JedisConnectionObject(redisHost, redisPort); + isSetup = setupRedisConnection(CHANNEL_PREFIX_STRING+"*"); + } + } catch (Exception e) { + isSubscribed = false; + isSetup = false; + logger.error("Fatal exception occurred attempting subscription: " + e.toString()); + logger.error("exception", e); + ++attempts; + } + } + } + + private boolean setupRedisConnection(final String channelRegEx) { + try { + isConnected = false; + if (null == subscriberJedis) subscriberJedis = jedisConn.getJedisResource(); + if (subscriberJedis != null) isConnected = true; + } catch (JedisConnectionException e) { + subscriberJedis = null; + isConnected = false; + logger.error("Fatal error! Couldn't establish connection to REDIS!"); + System.out.println("Fatal error! Couldn't establish connection to REDIS!"); + logger.error("exception", e); + } + if (isConnected) { + aidrSubscriber = new RedisSubscriber(); + jedisConn.setJedisSubscription(subscriberJedis, true); // we will be using pattern-based subscription + logger.info("Created new Jedis connection: " + subscriberJedis); + try { + subscribeToChannel(channelRegEx); + isSubscribed = true; + aidrSubscriber.setChannelName(channelRegEx); + logger.info("Resubscribed with pattern subscription: " + channelRegEx); + return true; + } catch (Exception e) { + isSubscribed = false; + logger.error("Fatal exception occurred attempting subscription: " + e.toString()); + System.out.println("Fatal exception occurred attempting subscription "); + logger.error("exception", e); + } + } + return false; + } + + + public void close() { + shutdownFlag = true; + stopSubscription(); + shutdownAndAwaitTermination(); + logger.info("All done, fetch service has been shutdown..."); + System.out.println("[close] All done, fetch service has been shutdown..."); + } + + + + // cleanup all threads + void shutdownAndAwaitTermination() { + int attempts = 0; + executorServicePool.shutdown(); // Disable new tasks from being submitted + while (!executorServicePool.isTerminated() && attempts < 3) { + try { + // Wait a while for existing tasks to terminate + if (!executorServicePool.awaitTermination(5, TimeUnit.SECONDS)) { + executorServicePool.shutdownNow(); // Cancel currently executing tasks + // Wait a while for tasks to respond to being cancelled + if (!executorServicePool.awaitTermination(5, TimeUnit.SECONDS)) + logger.error("Executor Thread Pool did not terminate"); + } else { + logger.info("All tasks completed post service shutdown"); + } + } catch (InterruptedException e) { + // (Re-)Cancel if current thread also interrupted + executorServicePool.shutdownNow(); + // Preserve interrupt status + Thread.currentThread().interrupt(); + } finally { + executorServicePool.shutdownNow(); + } + ++attempts; + if (!executorServicePool.isTerminated()) { + logger.warn("Warning! Some threads not shutdown still. Trying again, attempt = " + attempts); + } + } + } + + + private String getBinNumber(float confidence) { + int bin = 0; + for (int i = 10; i <= 100; i += 10) { + if (i > (confidence * 100)) { + return Integer.toString(bin); + } + ++bin; + } + return Integer.toString(bin - 1); + } + + + + //////////////////////////////////////////////////// + private class RedisSubscriber extends JedisPubSub { + + private String channelName = null; + + public void setChannelName(String channelName) { + this.channelName = channelName; + } + + public String getChannelName() { + return this.channelName; + } + + @Override + public void onMessage(String channel, String message) {} + + @Override + public void onPMessage(String pattern, String channel, String message) { + try { + if (!redisLoadShedder.containsKey(channel)) { + redisLoadShedder.put(channel, new LoadShedder(PERSISTER_LOAD_LIMIT, PERSISTER_LOAD_CHECK_INTERVAL_MINUTES, true)); + logger.info("Created new redis load shedder for channel: " + channel); + } + if (redisLoadShedder.get(channel).canProcess(channel)) { + manageChannelBuffers(pattern, channel, message); + } + } catch (Exception e) { + logger.error("Exception occurred, redisLoadShedder = " + redisLoadShedder + ", channel status: " + redisLoadShedder.containsKey(channel)); + System.out.println("Exception occurred, redisLoadShedder = " + redisLoadShedder + ", channel status: " + redisLoadShedder.containsKey(channel)); + logger.error("exception", e); + } + } + + @Override + public void onSubscribe(String channel, int subscribedChannels) { + logger.info("Subscribed to channel:" + channel); + } + + @Override + public void onUnsubscribe(String channel, int subscribedChannels) { + logger.info("Unsubscribed from channel:" + channel); + } + + @Override + public void onPUnsubscribe(String pattern, int subscribedChannels) { + logger.info("Unsubscribed from channel pattern:" + pattern + ", shutdownFlag = " + shutdownFlag); + } + + @Override + public void onPSubscribe(String pattern, int subscribedChannels) { + logger.info("Subscribed to channel pattern:" + pattern); + } + } + + + +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ConfCounterKey.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ConfCounterKey.java index ff10871dc..f8d7d1035 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ConfCounterKey.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ConfCounterKey.java @@ -17,7 +17,7 @@ public ConfCounterKey() { } public ConfCounterKey(String crisisCode, String attributeCode, String labelCode, String binNumber) { - super(); + super(crisisCode, attributeCode, labelCode); if (binNumber != null) { this.setBinNumber(binNumber); } @@ -57,4 +57,12 @@ public boolean equals(Object obj) { return true; } + + @Override + public String toString() { + return "ConfCounterKey [binNumber=" + binNumber + ", getCrisisCode()=" + + getCrisisCode() + ", getAttributeCode()=" + + getAttributeCode() + ", getLabelCode()=" + getLabelCode() + + "]"; + } } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CounterKey.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CounterKey.java index 6a03feb52..2420c86e3 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CounterKey.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CounterKey.java @@ -10,7 +10,6 @@ */ public class CounterKey implements Serializable { - private static final long serialVersionUID = 9168204241878331959L; private String crisisCode = null; @@ -65,7 +64,6 @@ public int hashCode() { return result; } - @Override public boolean equals(Object obj) { if (this == obj) @@ -92,4 +90,11 @@ public boolean equals(Object obj) { return false; return true; } + + @Override + public String toString() { + return "CounterKey [crisisCode=" + crisisCode + ", attributeCode=" + + attributeCode + ", labelCode=" + labelCode + "]"; + } + } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java new file mode 100644 index 000000000..8b9f3a382 --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java @@ -0,0 +1,254 @@ +/** + * This class provides the connection pool and methods to request and return + * JEDIS resources/connections to and from a JEDIS pool. + */ +package qa.qcri.aidr.analysis.utils; + +import java.util.Iterator; +import java.util.concurrent.ConcurrentHashMap; + + + + +import org.apache.log4j.Logger; + +import qa.qcri.aidr.common.logging.ErrorLog; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; +import redis.clients.jedis.exceptions.JedisConnectionException; + +public class JedisConnectionObject { + // Connection management related + public static ConcurrentHashMap allotedJedis; // active Jedis resources and their subscription type + + // Jedis related + private static JedisPoolConfig poolConfig = null; // only one JedisPoolConfig per servlet instance + private static JedisPool pool = null; // only one JedisPool per servlet instance + private Jedis subscriberJedis = null; // one for each GET request + + private static OutputConfigurator configProperties = OutputConfigurator.getInstance(); + public static String redisHost = configProperties.getProperty(OutputConfigurationProperty.REDIS_HOST); + public static int redisPort = Integer.valueOf(configProperties.getProperty(OutputConfigurationProperty.REDIS_PORT)); + + private static boolean poolSetup = false; + private boolean connectionSetup = false; + + // Logger setup + private static Logger logger = Logger.getLogger(JedisConnectionObject.class); + private static ErrorLog elog = new ErrorLog(); + /** + * + * @param host hostname on which REDIS resides + * @param port port number to use for establishing connection + */ + public JedisConnectionObject(final String host, final int port) { + // For now: set up a simple configuration that logs on the console + //PropertyConfigurator.configure("log4j.properties"); // where to place the properties file? + //BasicConfigurator.configure(); // initialize log4j logging + //System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "INFO"); // set logging level for slf4j + + allotedJedis = new ConcurrentHashMap(); + redisHost = host; + redisPort = port; + //synchronized (allotedJedis) + { + if (null == JedisConnectionObject.poolConfig) { + JedisConnectionObject.poolConfig = new JedisPoolConfig(); + JedisConnectionObject.poolConfig.setMaxTotal(200); + JedisConnectionObject.poolConfig.setMaxIdle(50); + JedisConnectionObject.poolConfig.setMinIdle(10); + JedisConnectionObject.poolConfig.setMaxWaitMillis(3000); + //JedisConnectionObject.poolConfig.setMaxActive(200); + //poolConfig.setTestWhileIdle(true); + //poolConfig.setTestOnBorrow(true); + //poolConfig.setTestOnReturn(true); + //poolConfig.numTestsPerEvictionRun = 10; + //poolConfig.timeBetweenEvictionRunsMillis = 3000; + //JedisConnectionObject.poolConfig.whenExhaustedAction = org.apache.commons.pool.impl.GenericKeyedObjectPool.WHEN_EXHAUSTED_GROW; + + logger.info("New Jedis poolConfig: " + JedisConnectionObject.poolConfig); + } else { + logger.info("Reusing existing Jedis poolConfig: " + JedisConnectionObject.poolConfig); + } + if (null == JedisConnectionObject.pool) { + try { + JedisConnectionObject.pool = new JedisPool(JedisConnectionObject.poolConfig, redisHost, redisPort, 30000); + JedisConnectionObject.poolSetup = true; + logger.info("New Jedis pool: " + pool); + } catch (Exception e) { + logger.error("Fatal error! Could not initialize Jedis Pool!"); + logger.error(elog.toStringException(e)); + JedisConnectionObject.poolConfig = null; + JedisConnectionObject.pool = null; + JedisConnectionObject.poolSetup = false; + } + } else { + JedisConnectionObject.poolSetup = true; + logger.info("Reusing existing Jedis pool: " + JedisConnectionObject.pool); + } + //allotedJedis.notifyAll(); + } + } + + /** + * Connects to a REDIS DB on default port 6379 of localhost + */ + public JedisConnectionObject() { + this(redisHost, redisPort); + } + + /** + * Connects to a REDIS DB on lcoalhost with specified port + * @param port port number to use for connecting to REDIS DB + */ + public JedisConnectionObject(final int port) { + this(redisHost, port); + } + + /** + * Get a connection resource from JEDIS pool + * @return + */ + public Jedis getJedisResource() { + Jedis subscriberJedis = null; + if (isPoolSetup()) { + try { + subscriberJedis = JedisConnectionObject.pool.getResource(); + connectionSetup = true; + } catch (Exception e) { + subscriberJedis = null; + connectionSetup = false; + logger.error("Fatal error! Could not get a resource from the pool."); + logger.error(elog.toStringException(e)); + } + if (subscriberJedis != null) { + allotedJedis.put(subscriberJedis, false); // initially nothing assigned + logger.info("Allocating jedis resource to caller: " + subscriberJedis); + return subscriberJedis; + } + } + return null; + } + + /** + * @param jedis Jedis connection resource + * @param patternFlag subscription type associated with the Jedis connection + * resource: true = pattern subscription, false = single channel subscription + */ + public void setJedisSubscription(Jedis jedis, boolean patternFlag) { + if (jedis != null) { + allotedJedis.put(jedis, patternFlag); + } + } + + /** + * @param jedis Jedis connection resource + * @return subscription type associated with the Jedis connection + * resource: true = pattern subscription, false = single channel subscription + */ + public boolean getJedisSubscription(Jedis jedis) { + if (jedis != null) { + return allotedJedis.get(jedis); + } + return false; + } + + /** + * Returns to the Jedis pool a specific instance of allocated Jedis connection resource + * @param jedisInstance a Jedis instance borrowed from the Jedis Pool + */ + public void returnJedis(Jedis jedisInstance) { + if (JedisConnectionObject.pool != null) { + try { + if (null != jedisInstance && jedisInstance.isConnected()) { + if (allotedJedis != null) allotedJedis.remove(jedisInstance); + logger.info("[returnJedis] Returned jedis resource: " + jedisInstance); + jedisInstance.close(); + jedisInstance = null; + //JedisConnectionObject.pool.returnResource(jedisInstance); + connectionSetup = false; + } + } catch (JedisConnectionException e) { + logger.error("JedisConnectionException occurred..."); + logger.error(elog.toStringException(e)); + if (jedisInstance != null && jedisInstance.isConnected()) jedisInstance.close(); + jedisInstance = null; + //JedisConnectionObject.pool.returnBrokenResource(jedisInstance); + connectionSetup = false; + } + } + } + + /** + * + * @return active Jedis pool object + */ + public JedisPool getJedisPool() { + return JedisConnectionObject.pool; + } + + /** + * + * @return active Jedis poolconfig object + */ + public JedisPoolConfig getJedisPoolConfig() { + return JedisConnectionObject.poolConfig; + } + + /** + * + * @return Redis hostname being used + */ + public String getRedisHost() { + return redisHost; + } + + /** + * + * @return port number being used to connect to Redis + */ + public int getRedisPort() { + return redisPort; + } + + /** + * Returns the state of Jedis pool for this JedisConnectionObject instance + * @return true is Jedis pool has been established, false otherwise + */ + public boolean isPoolSetup() { + return JedisConnectionObject.poolSetup; + } + + /** + * Returns the state of Jedis connection for this JedisConnectionObject instance + * @return true is Jedis connection has been established, false otherwise + */ + public boolean isConnectionSetup() { + return connectionSetup; + } + + /** + * Closes all open jedis connections and destroys the Jedis pool. + * Warning! Not thread safe! Use with care! + */ + public void closeAll() { + if (allotedJedis != null) { + Iteratoritr = allotedJedis.keySet().iterator(); + while (itr.hasNext()) { + //stopSubscription(this.aidrSubscriber, this.subscriberJedis); + Jedis j = itr.next(); + returnJedis(j); + } + allotedJedis.clear(); + allotedJedis = null; + } + if (JedisConnectionObject.pool != null && JedisConnectionObject.poolConfig != null) { + JedisConnectionObject.pool.destroy(); + JedisConnectionObject.pool = null; + JedisConnectionObject.poolConfig = null; + logger.info("Pool destroyed"); + } + JedisConnectionObject.poolSetup = false; + } +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java new file mode 100644 index 000000000..aaa32d2eb --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java @@ -0,0 +1,24 @@ +package qa.qcri.aidr.analysis.utils; + +import qa.qcri.aidr.common.code.ConfigurationProperty; + +public enum OutputConfigurationProperty implements ConfigurationProperty { + + REDIS_HOST("REDIS_HOST"), REDIS_PORT("REDIS_PORT"), LOGGER("logger"), MANAGER_URL( + "managerUrl"), PERSISTER_LOAD_LIMIT("PERSISTER_LOAD_LIMIT"), PERSISTER_LOAD_CHECK_INTERVAL_MINUTES( + "PERSISTER_LOAD_CHECK_INTERVAL_MINUTES"), OUTPUT_REST_URI( + "OUTPUT_REST_URI"), MAX_MESSAGES_COUNT("MAX_MESSAGES_COUNT"), TAGGER_CHANNEL_BASENAME( + "TAGGER_CHANNEL_BASENAME"); + + private final String configurationProperty; + + private OutputConfigurationProperty(String property) { + configurationProperty = property; + } + + @Override + public String getName() { + return this.configurationProperty; + } + +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurator.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurator.java new file mode 100644 index 000000000..2e1492845 --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurator.java @@ -0,0 +1,35 @@ +package qa.qcri.aidr.analysis.utils; + +import org.apache.log4j.Logger; + +import qa.qcri.aidr.common.code.impl.BaseConfigurator; +import qa.qcri.aidr.common.exception.ConfigurationPropertyFileException; +import qa.qcri.aidr.common.exception.ConfigurationPropertyNotRecognizedException; +import qa.qcri.aidr.common.exception.ConfigurationPropertyNotSetException; + +/** + * User: yakubenkova.elena@gmail.com Date: 29.09.14 + */ + +public class OutputConfigurator extends BaseConfigurator { + + private static final Logger LOGGER = Logger + .getLogger(OutputConfigurator.class); + + public static final String configLoadFileName = "config.properties"; + + private static final OutputConfigurator instance = new OutputConfigurator(); + + private OutputConfigurator() { + LOGGER.info("Initializing OutputConfigurator."); + this.initProperties(configLoadFileName, OutputConfigurationProperty.values()); + } + + public static OutputConfigurator getInstance() + throws ConfigurationPropertyNotSetException, + ConfigurationPropertyNotRecognizedException, + ConfigurationPropertyFileException { + return instance; + } + +} diff --git a/aidr-analytics/src/main/resources/config.properties b/aidr-analytics/src/main/resources/config.properties new file mode 100644 index 000000000..e8e47e48f --- /dev/null +++ b/aidr-analytics/src/main/resources/config.properties @@ -0,0 +1,35 @@ +# +# Configuration for aidr-output +# + +############################# +# MANDATORY configuration +# Values must be set at each installation +############################# + +# Redis hostname +REDIS_HOST=${REDIS_HOST} + +#REST URI of AIDR output +OUTPUT_REST_URI=http://${OUTPUT_API_HOST}:${OUTPUT_API_PORT}/${OUTPUT_API_WEB_APP_CONTEXT}/rest/ + +############################# +# OPTIONAL configuration +# Values can be left at their default values +############################# + +# Redis port number (default: 6379) +REDIS_PORT=${REDIS_PORT} +logger=log4j + +#Maximum number of messages that can be returned from the buffer +MAX_MESSAGES_COUNT=1000 + +############################# +# INHERITED configuration +# Values are inherited from common configuration +############################# +TAGGER_CHANNEL_BASENAME=${TAGGER_CHANNEL_BASENAME} +managerUrl=http://${MANAGER_HOST}:${MANAGER_PORT}/${MANAGER_WEB_APP_CONTEXT} +PERSISTER_LOAD_LIMIT=${PERSISTER_LOAD_LIMIT} +PERSISTER_LOAD_CHECK_INTERVAL_MINUTES=${PERSISTER_LOAD_CHECK_INTERVAL_MINUTES} diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java index 9a6e0651b..b4a97f66e 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java @@ -62,7 +62,7 @@ public ClassifiedFilteredTweet deserialize(String rawJsonString) { if (null == rawJsonString) { return null; } - + if (rawJsonString.startsWith("[")) // should never happen rawJsonString = rawJsonString.substring(1, rawJsonString.length()-1); @@ -133,19 +133,21 @@ public ClassifiedFilteredTweet deserialize(String rawJsonString) { for (int i = 0;i < nominalLabels.size();i++) { NominalLabel nLabel = new NominalLabel(); JsonObject temp = (JsonObject) nominalLabels.get(i); - - nLabel.attribute_code = (temp.has("attribute_code") && !temp.get("attribute_code").isJsonNull()) ? temp.get("attribute_code").getAsString() : null; - nLabel.label_code = (temp.has("label_code") && !temp.get("label_code").isJsonNull()) ? temp.get("label_code").getAsString() : null; - nLabel.confidence = (temp.has("confidence") && !temp.get("confidence").isJsonNull()) ? temp.get("confidence").getAsFloat() : 0; - - nLabel.attribute_name = (temp.has("attribute_name") && !temp.get("attribute_name").isJsonNull()) ? temp.get("attribute_name").getAsString() : null; - nLabel.label_name = (temp.has("label_name") && !temp.get("label_name").isJsonNull()) ? temp.get("label_name").getAsString() : null; - nLabel.attribute_description = (temp.has("attribute_description") && !temp.get("attribute_description").isJsonNull()) ? temp.get("attribute_description").getAsString() : null; - nLabel.label_description = (temp.has("label_description") && !temp.get("label_description").isJsonNull())? temp.get("label_description").getAsString() : null; - nLabel.from_human = (temp.has("from_human") && !temp.get("from_human").isJsonNull()) ? temp.get("from_human").getAsBoolean() : false; - - jsonObj.nominal_labels.add(nLabel); - nominal_labels.add(nLabel); + + if (!temp.get("attribute_code").isJsonNull() && !temp.get("label_code").isJsonNull()) { + nLabel.attribute_code = (temp.has("attribute_code") && !temp.get("attribute_code").isJsonNull()) ? temp.get("attribute_code").getAsString() : null; + nLabel.label_code = (temp.has("label_code") && !temp.get("label_code").isJsonNull()) ? temp.get("label_code").getAsString() : null; + nLabel.confidence = (temp.has("confidence") && !temp.get("confidence").isJsonNull()) ? temp.get("confidence").getAsFloat() : 0; + + nLabel.attribute_name = (temp.has("attribute_name") && !temp.get("attribute_name").isJsonNull()) ? temp.get("attribute_name").getAsString() : null; + nLabel.label_name = (temp.has("label_name") && !temp.get("label_name").isJsonNull()) ? temp.get("label_name").getAsString() : null; + nLabel.attribute_description = (temp.has("attribute_description") && !temp.get("attribute_description").isJsonNull()) ? temp.get("attribute_description").getAsString() : null; + nLabel.label_description = (temp.has("label_description") && !temp.get("label_description").isJsonNull())? temp.get("label_description").getAsString() : null; + nLabel.from_human = (temp.has("from_human") && !temp.get("from_human").isJsonNull()) ? temp.get("from_human").getAsBoolean() : false; + + jsonObj.nominal_labels.add(nLabel); + nominal_labels.add(nLabel); + } } return jsonObj; } @@ -156,11 +158,11 @@ public ClassifiedFilteredTweet deserialize(String rawJsonString) { return null; } } - + public void setCreateAt(final Date created_at) { this.created_at = created_at; } - + public Date getCreatedAt() { return created_at; } From 42b336721a5b269565c8ea8b205aef73ed15469b Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Mon, 27 Jul 2015 18:52:14 +0300 Subject: [PATCH 06/44] reduced debug msgs --- .../analysis/api/WriteStatisticsData.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java index 11f0f11db..a01cb1f32 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -136,15 +136,13 @@ private ReturnCode writeOutputDataToTagDataDB(Long granularity,Long timestamp) { // tagDataMap.size() : "null")); try { for (CounterKey key : cbManager.getTagDataMap().keySet()) { - System.out.println("Will attempt persistence of tag key: " + key.toString()); TagDataMapRecord tCount = (TagDataMapRecord) ChannelBufferManager.getTagDataMap().get(key); TagData t = new TagData(key.getCrisisCode(), timestamp, granularity, key.getAttributeCode(), key.getLabelCode(), tCount.getCount(granularity)); - tagDataEJB.writeData(t); + if (tCount.getCount(granularity) > 0) { + System.out.println("Will attempt persistence of tag key: " + key.toString()); + tagDataEJB.writeData(t); + } tCount.resetCount(granularity); - // TagDataMapRecord temp = (TagDataMapRecord) tagDataMap.get(key); - // logger.info("[writeOutputDataToTagDataDB] After reset, count for key = " - // + key + " is = " + temp.getCount(granularity)); - } } catch (Exception e) { logger.error("Error in writing to TagDataDB table!", e); @@ -156,11 +154,13 @@ private ReturnCode writeOutputDataToTagDataDB(Long granularity,Long timestamp) { private ReturnCode writeOutputDataToConfDataDB(Long granularity, Long timestamp) { try { for (CounterKey key : cbManager.getConfDataMap().keySet()) { - System.out.println("Will attempt persistence of conf key: " + key.toString()); ConfDataMapRecord fCount = (ConfDataMapRecord) ChannelBufferManager.getConfDataMap().get(key); ConfidenceData f = new ConfidenceData(key.getCrisisCode(), timestamp, granularity, key.getAttributeCode(), key.getLabelCode(), Integer.parseInt(((ConfCounterKey) key).getBinNumber()), fCount.getCount(granularity)); - confDataEJB.writeData(f); + if (fCount.getCount(granularity) > 0) { + System.out.println("Will attempt persistence of conf key: " + key.toString()); + confDataEJB.writeData(f); + } fCount.resetCount(granularity); } } catch (Exception e) { @@ -195,19 +195,19 @@ public void run() { // Write to DB table ReturnCode retVal = writeOutputDataToTagDataDB(granularity, currentTime); lastTagDataWriteTime.put(granularity, currentTime); - logger.info("retVal = " + retVal); + //logger.info("retVal = " + retVal); if (ReturnCode.SUCCESS.equals(retVal)) { - logger.info("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); - System.out.println("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); + //logger.info("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); + //System.out.println("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); } } if (0 == lastConfDataWriteTime.get(granularity) || (currentTime - lastConfDataWriteTime.get(granularity)) >= granularity) { // Write to DB table ReturnCode retVal = writeOutputDataToConfDataDB(granularity, currentTime); - logger.info("retVal = " + retVal); + //logger.info("retVal = " + retVal); lastConfDataWriteTime.put(granularity, currentTime); if (ReturnCode.SUCCESS.equals(retVal)) { - logger.info("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); + //logger.info("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); } } } From 22faeb50901b2f15d00967920ef7dac9f3f9f8ff Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Tue, 28 Jul 2015 13:47:21 +0530 Subject: [PATCH 07/44] Minor changes --- .../qa/qcri/aidr/common/code/Configurator.java | 4 +++- .../aidr/common/code/impl/BaseConfigurator.java | 17 +++++++++++++++++ .../qa/qcri/aidr/common/util/EmailClient.java | 6 +++--- .../qcri/aidr/utils/PersisterConfigurator.java | 14 +------------- .../qcri/aidr/predictui/api/MiscResource.java | 9 ++++++++- .../aidr/predict/common/TaggerConfigurator.java | 14 +------------- 6 files changed, 33 insertions(+), 31 deletions(-) diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/code/Configurator.java b/aidr-common/src/main/java/qa/qcri/aidr/common/code/Configurator.java index 457ab40f7..cf9c06326 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/code/Configurator.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/code/Configurator.java @@ -3,6 +3,7 @@ import qa.qcri.aidr.common.exception.ConfigurationPropertyFileException; import qa.qcri.aidr.common.exception.ConfigurationPropertyNotRecognizedException; import qa.qcri.aidr.common.exception.ConfigurationPropertyNotSetException; +import qa.qcri.aidr.common.exception.DirectoryNotWritableException; public interface Configurator { @@ -13,6 +14,7 @@ public void initProperties(String configLoadFileName, ConfigurationPropertyFileException; public String getProperty(ConfigurationProperty property); - + public String getProperty(String propertyName); public void setProperty(String property, String newValue); + public void directoryIsWritable(String propertyName) throws DirectoryNotWritableException; } diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/code/impl/BaseConfigurator.java b/aidr-common/src/main/java/qa/qcri/aidr/common/code/impl/BaseConfigurator.java index 7f5bd50fe..a74ef3b0a 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/code/impl/BaseConfigurator.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/code/impl/BaseConfigurator.java @@ -1,5 +1,6 @@ package qa.qcri.aidr.common.code.impl; +import java.io.File; import java.util.Map; import org.apache.log4j.Logger; @@ -10,6 +11,7 @@ import qa.qcri.aidr.common.exception.ConfigurationPropertyFileException; import qa.qcri.aidr.common.exception.ConfigurationPropertyNotRecognizedException; import qa.qcri.aidr.common.exception.ConfigurationPropertyNotSetException; +import qa.qcri.aidr.common.exception.DirectoryNotWritableException; public abstract class BaseConfigurator implements Configurator { @@ -35,6 +37,11 @@ public String getProperty(ConfigurationProperty property) { return propertyMap.get(property.getName()); } + @Override + public String getProperty(String propertyName) { + return propertyMap.get(propertyName); + } + @Override public void setProperty(String property, String newValue) { if(propertyMap.containsKey(property)){ @@ -48,4 +55,14 @@ public void setProperty(String property, String newValue) { property, null); } } + + @Override + public void directoryIsWritable(String propertyName) throws DirectoryNotWritableException{ + String directoryLocation = this.getProperty(propertyName); + File f = new File(directoryLocation); + if(!f.canWrite()) { + LOGGER.info(propertyName+ " = " +directoryLocation+ " is not writable. Please verify if this is a valid writable directory."); + throw new DirectoryNotWritableException(propertyName, directoryLocation); + } + } } diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java b/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java index ace1468a5..4662a0f7e 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java @@ -22,7 +22,7 @@ public class EmailClient { private static CommonConfigurator configProperties = CommonConfigurator.getInstance(); private static Logger logger = Logger.getLogger(EmailClient.class.getName()); - public static void sendErrorMail(String module, String code, String errorMsg ) + public static void sendErrorMail(String subject, String body ) { // Recipient's email IDs. String recipients = configProperties.getProperty(CommonConfigurationProperty.RECIPIENT_EMAIL); @@ -50,13 +50,13 @@ protected PasswordAuthentication getPasswordAuthentication() { MimeMessage message = new MimeMessage(session); // Set Subject: header field - message.setSubject(module); + message.setSubject(subject); Date d = new Date(System.currentTimeMillis()); String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); // Now set the actual message - message.setText(time+" "+module +" "+ code +"\n"+ errorMsg); + message.setText(body); // Set From: header field of the header. message.setFrom(new InternetAddress(sender,"AIDR Admin")); diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterConfigurator.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterConfigurator.java index 5b6095b85..8387d0fd5 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterConfigurator.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterConfigurator.java @@ -1,7 +1,5 @@ package qa.qcri.aidr.utils; -import java.io.File; - import org.apache.log4j.Logger; import qa.qcri.aidr.common.code.impl.BaseConfigurator; @@ -22,20 +20,10 @@ private PersisterConfigurator() throws DirectoryNotWritableException { LOGGER.info("Instantiating PersisterConfigurator."); this.initProperties(configLoadFileName, PersisterConfigurationProperty.values()); - this.directoryIsWritable(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH); + this.directoryIsWritable(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH.getName()); } public static PersisterConfigurator getInstance() { return instance; } - - private void directoryIsWritable(PersisterConfigurationProperty propertyName) throws DirectoryNotWritableException{ - String directoryLocation = this.getProperty(propertyName); - File f = new File(directoryLocation); - if(!f.canWrite()) { - LOGGER.info(propertyName.getName()+ " = " +directoryLocation+ " is not writable. Please verify if this is a valid writable directory."); - throw new DirectoryNotWritableException(propertyName.getName(), directoryLocation); - } - } - } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java index b4e13e7aa..6d3c89a33 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java @@ -4,6 +4,9 @@ */ package qa.qcri.aidr.predictui.api; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; import java.util.List; import qa.qcri.aidr.common.util.EmailClient; @@ -256,7 +259,11 @@ public Response downloadHumanLabeledDocumentsByCrisisIDUserName(String queryStri public Response sendErrorEmail(@FormParam("code") String code, @FormParam("module") String module, @FormParam("description") String description) throws Exception { Boolean emailSent = true; try { - EmailClient.sendErrorMail(module, code, description); + Date d = new Date(System.currentTimeMillis()); + String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); + + String body = time+" "+module +" "+ code +"\n"+ description; + EmailClient.sendErrorMail(module, body); } catch (Exception e) { logger.error("Unable to send email"); logger.error(e.getMessage()); diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerConfigurator.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerConfigurator.java index b3e7018f9..72aecb70b 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerConfigurator.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerConfigurator.java @@ -1,12 +1,9 @@ package qa.qcri.aidr.predict.common; -import java.io.File; - import org.apache.log4j.Logger; import qa.qcri.aidr.common.code.Configurator; import qa.qcri.aidr.common.code.impl.BaseConfigurator; -import qa.qcri.aidr.common.exception.DirectoryNotWritableException; /** * User: yakubenkova.elena@gmail.com Date: 29.09.14 @@ -24,19 +21,10 @@ private TaggerConfigurator() { LOGGER.info("Instantiating TaggerConfigurator,"); this.initProperties(configLoadFileName, TaggerConfigurationProperty.values()); - this.directoryIsWritable(TaggerConfigurationProperty.MODEL_STORE_PATH); + this.directoryIsWritable(TaggerConfigurationProperty.MODEL_STORE_PATH.getName()); } public static Configurator getInstance() { return instance; } - - private void directoryIsWritable(TaggerConfigurationProperty propertyName) throws DirectoryNotWritableException{ - String directoryLocation = this.getProperty(propertyName); - File f = new File(directoryLocation); - if(!f.canWrite()) { - LOGGER.info(propertyName.getName()+ " = " +directoryLocation+ " is not writable. Please verify if this is a valid writable directory."); - throw new DirectoryNotWritableException(propertyName.getName(), directoryLocation); - } - } } \ No newline at end of file From f609a994988478d8a9961835c5cd40835b7d2f28 Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Tue, 28 Jul 2015 16:58:06 +0530 Subject: [PATCH 08/44] Commit for #98777058 --- aidr-tagger/pom.xml | 12 +++-- .../java/qa/qcri/aidr/predict/Controller.java | 5 +- .../java/qa/qcri/aidr/predict/DataStore.java | 46 ++++++------------- .../communication/HttpInputManager.java | 7 +-- .../communication/HttpInputWorker.java | 22 ++++----- .../communication/HttpOutputManager.java | 10 ++-- .../communication/HttpOutputWorker.java | 10 +--- .../predict/communication/OutputFilter.java | 3 -- .../predict/communication/OutputMatcher.java | 21 ++++----- .../featureextraction/FeatureExtractor.java | 9 ++-- .../predict/featureextraction/WordSet.java | 4 -- .../qcri/aidr/predict/TaggerTesterHelper.java | 2 + .../qcri/aidr/predict/TaggerTesterTest.java | 4 +- 13 files changed, 55 insertions(+), 100 deletions(-) diff --git a/aidr-tagger/pom.xml b/aidr-tagger/pom.xml index a7bf336a3..38a30060e 100644 --- a/aidr-tagger/pom.xml +++ b/aidr-tagger/pom.xml @@ -8,6 +8,11 @@ 1.0 Reads collected items from a Redis queue, annotates them using an automatic classifier, and writes them to another Redis queue http://aidr.qcri.org/ + + UTF-8 + ${basedir}/src/main/resources + true + @@ -16,7 +21,7 @@ maven-surefire-plugin 2.17 - True + ${skipTests} **/TaggerTesterTest.java @@ -273,10 +278,7 @@ - - UTF-8 - ${basedir}/src/main/resources - + diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/Controller.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/Controller.java index 1f531a3dc..0553cb51e 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/Controller.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/Controller.java @@ -11,7 +11,6 @@ import qa.qcri.aidr.predict.common.TaggerConfigurator; import qa.qcri.aidr.predict.communication.*; import qa.qcri.aidr.predict.featureextraction.FeatureExtractor; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.redis.LoadShedder; /** @@ -35,7 +34,6 @@ public class Controller { // Debugging private static Logger logger = Logger.getLogger(Controller.class); - private static ErrorLog elog = new ErrorLog(); public static void main(String[] args) { @@ -150,8 +148,7 @@ public void run() { Thread.sleep(10000); } catch (InterruptedException e) { e.printStackTrace(); - logger.error("Exception in starting tagger Controller!"); - logger.error(elog.toStringException(e)); + logger.error("Exception in starting tagger Controller!", e); } } } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/DataStore.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/DataStore.java index 4fe91e47f..8b591d25b 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/DataStore.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/DataStore.java @@ -1,6 +1,5 @@ package qa.qcri.aidr.predict; -import java.io.Serializable; import java.sql.Connection; import java.sql.Driver; import java.sql.DriverManager; @@ -26,13 +25,12 @@ import org.apache.log4j.Logger; import org.json.JSONObject; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.DocumentDTO; import qa.qcri.aidr.dbmanager.dto.DocumentNominalLabelDTO; import qa.qcri.aidr.dbmanager.dto.DocumentNominalLabelIdDTO; import qa.qcri.aidr.predict.classification.nominal.Model; -import qa.qcri.aidr.predict.classification.nominal.NominalLabelBC; import qa.qcri.aidr.predict.classification.nominal.ModelNominalLabelPerformance; +import qa.qcri.aidr.predict.classification.nominal.NominalLabelBC; import qa.qcri.aidr.predict.common.Helpers; import qa.qcri.aidr.predict.common.TaggerConfigurationProperty; import qa.qcri.aidr.predict.common.TaggerConfigurator; @@ -63,7 +61,6 @@ public class DataStore { public static TaskManagerRemote taskManager = null; private static Logger logger = Logger.getLogger(DataStore.class); - private static ErrorLog elog = new ErrorLog(); private static final String remoteEJBJNDIName = TaggerConfigurator .getInstance().getProperty( @@ -136,9 +133,7 @@ public static void initTaskManager() { logger.info("Success in connecting to remote EJB to initialize taskManager"); } } catch (NamingException e) { - logger.error("Error in JNDI lookup for initializing remote EJB"); - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("Error in JNDI lookup for initializing remote EJB", e); } } @@ -292,8 +287,7 @@ public static void close(Connection con) { try { con.close(); } catch (SQLException e) { - logger.error("Exception when returning MySQL connection"); - logger.error(elog.toStringException(e)); + logger.error("Exception when returning MySQL connection", e); } } @@ -304,8 +298,7 @@ public static void close(Statement statement) { try { statement.close(); } catch (SQLException e) { - logger.error("Could not close statement"); - logger.error(elog.toStringException(e)); + logger.error("Could not close statement", e); } } @@ -316,8 +309,7 @@ public static void close(ResultSet resultset) { try { resultset.close(); } catch (SQLException e) { - logger.error("Could not close statement"); - logger.error(elog.toStringException(e)); + logger.error("Could not close statement", e); } } @@ -335,8 +327,7 @@ public static Integer getNullLabelID(int attributeID) { return result.getInt(1); } } catch (SQLException ex) { - logger.error("Error in executing SQL statement: " + sql); - logger.error(elog.toStringException(ex)); + logger.error("Error in executing SQL statement: " + sql, ex); } finally { close(result); close(query); @@ -389,11 +380,9 @@ static void getLabeledSet(String sql, ArrayList wordVectors, .getJSONArray("words"))); } } catch (SQLException e) { - logger.error("Exception while fetching dataset"); - logger.error(elog.toStringException(e)); + logger.error("Exception while fetching dataset. ", e); } catch (Exception e) { - logger.error("Exception while fetching dataset"); - logger.error(elog.toStringException(e)); + logger.error("Exception while fetching dataset", e); } finally { close(result); close(statement); @@ -559,9 +548,7 @@ public static void saveDocumentsToDatabase(List items) { saveNewDocumentsCount = 0; } } catch (Exception e) { - logger.error("Exception when attempting to write Document to database"); - e.printStackTrace(); - logger.error(elog.toStringException(e)); + logger.error("Exception when attempting to write Document to database", e); } saveHumanLabels(items); } @@ -617,8 +604,7 @@ static void saveHumanLabels(List documents) { sendNewLabeledDocumentNotification(notifications); } catch (Exception e) { - logger.error("Exception when attempting to insert new document labels"); - logger.error(elog.toStringException(e)); + logger.error("Exception when attempting to insert new document labels", e); } } @@ -738,8 +724,7 @@ public static ArrayList getActiveModels() { } } catch (SQLException e) { - logger.error("Exception when getting model state"); - logger.error(elog.toStringException(e)); + logger.error("Exception when getting model state", e); } finally { close(result); close(sql); @@ -759,7 +744,6 @@ public static void deleteModel(int modelID) { sql.executeUpdate(); } catch (SQLException e) { logger.error("Exception while deleting model"); - logger.error(elog.toStringException(e)); } finally { close(sql); close(conn); @@ -781,7 +765,6 @@ public static HashMap getCrisisIDs() { } } catch (SQLException e) { logger.error("Exception when getting crisis IDs", e); - logger.error(elog.toStringException(e)); } finally { close(result); close(sql); @@ -871,8 +854,7 @@ public static int saveModelToDatabase(int crisisID, int nominalAttributeID, + selectModelFamilyID + ")"); mfUpdate.executeUpdate(); } catch (SQLException e) { - logger.error("Exception while saving model to database"); - logger.error(elog.toStringException(e)); + logger.error("Exception while saving model to database", e); } finally { close(result); close(modelInsert); @@ -916,7 +898,6 @@ public static HashMap> getNominalLabelTraining } } catch (SQLException e) { logger.error("Exception when getting nominal label training values", e); - logger.error(elog.toStringException(e)); } finally { close(result); close(sql); @@ -949,8 +930,7 @@ public static void saveClassifiedDocumentCounts(HashMap result = - new ObjectMapper().readValue(jsonResponse, HashMap.class); + objectMapper.readValue(jsonResponse, HashMap.class); if(result != null && result.get("crisisId") != null && result.get("crisisId") != new Integer(0)) { @@ -159,7 +159,7 @@ run the CLEANUP routine, and FAIL (forcing the user to run the tagger tester aga JsonParser jsonParser = new JsonParser(); JsonObject jsonObject = (JsonObject) jsonParser.parse(jsonResponse); - if( jsonObject != null && jsonObject.get("nominalAttributeId") != null ) { + if( jsonObject != null && jsonObject.get("nominalAttributeID") != null ) { nominalAttributeId = jsonObject.get("nominalAttributeID").getAsLong(); } From 3515fbd6e872407e24a6385f312be496eb84ca45 Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Tue, 28 Jul 2015 18:31:20 +0530 Subject: [PATCH 09/44] Commit for #98777744 --- .../aidr/output/getdata/ChannelBuffer.java | 9 ------ .../output/getdata/ChannelBufferManager.java | 30 +++++-------------- .../output/getdata/GetBufferedAIDRData.java | 7 ++--- .../qcri/aidr/output/stream/AsyncStream.java | 23 ++++---------- .../stream/AsyncStreamRedisSubscriber.java | 15 ++++------ .../aidr/output/utils/AIDROutputPing.java | 3 +- .../output/utils/JedisConnectionObject.java | 8 ----- .../aidr/output/utils/JsonDataFormatter.java | 2 -- .../output/utils/TaggerJsonOutputAdapter.java | 6 +--- aidr_testers.sh | 2 +- 10 files changed, 23 insertions(+), 82 deletions(-) diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBuffer.java b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBuffer.java index 56b34d8db..8d89fd7d2 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBuffer.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBuffer.java @@ -12,18 +12,13 @@ import java.util.ArrayList; import java.util.Date; -import java.util.Iterator; import java.util.List; import org.apache.commons.collections.Buffer; import org.apache.commons.collections.BufferUtils; import org.apache.commons.collections.buffer.CircularFifoBuffer; - - import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; - public class ChannelBuffer { public static int MAX_BUFFER_SIZE = 2000; // number of elements the buffer will hold at any time @@ -36,7 +31,6 @@ public class ChannelBuffer { private Boolean publiclyListed = true; private static Logger logger = Logger.getLogger(ChannelBuffer.class); - private static ErrorLog elog = new ErrorLog(); public ChannelBuffer(final String name, final int bufferSize) { this.channelName = name; @@ -76,7 +70,6 @@ public void addMessage(String msg) { } } catch (Exception e) { logger.error("Couldn't add message to buffer for channel: " + this.channelName); - logger.error(elog.toStringException(e)); } } @@ -90,7 +83,6 @@ public void addAllMessages(ArrayList msgList) { } } catch (Exception e) { logger.error("Couldn't add message to buffer for channel: " + this.channelName); - logger.error(elog.toStringException(e)); } } @@ -124,7 +116,6 @@ public List getMessages(int msgCount) { return returnList; } catch (Exception e) { logger.error("Error in creating list out of buffered messages"); - logger.error(elog.toStringException(e)); return null; } } diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java index 062df786b..379fbc057 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java @@ -38,7 +38,6 @@ import org.glassfish.jersey.jackson.JacksonFeature; import qa.qcri.aidr.common.code.Configurator; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.redis.LoadShedder; //import qa.qcri.aidr.output.utils.AIDROutputConfig; import qa.qcri.aidr.output.utils.JedisConnectionObject; @@ -58,7 +57,6 @@ public class ChannelBufferManager { private static int PERSISTER_LOAD_CHECK_INTERVAL_MINUTES; private static Logger logger = Logger.getLogger(ChannelBufferManager.class); - private static ErrorLog elog = new ErrorLog(); // Thread related private static ExecutorService executorServicePool = null; @@ -121,9 +119,7 @@ public void initiateChannelBufferManager(final String channelRegEx) { } catch (JedisConnectionException e) { subscriberJedis = null; isConnected = false; - logger.error("Fatal error! Couldn't establish connection to REDIS!"); - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("Fatal error! Couldn't establish connection to REDIS!", e); } if (isConnected) { aidrSubscriber = new RedisSubscriber(); @@ -137,7 +133,6 @@ public void initiateChannelBufferManager(final String channelRegEx) { } catch (Exception e) { isSubscribed = false; logger.error("Fatal exception occurred attempting subscription: " + e.toString()); - logger.error(elog.toStringException(e)); } if (isSubscribed) { subscribedChannels = new ConcurrentHashMap(); @@ -209,7 +204,7 @@ public void manageChannelBuffers(final String subscriptionPattern, final String } statusFlags.clear(); } catch (Exception e) { - logger.error(elog.toStringException(e)); + logger.error(e.getMessage()); } } lastPublicFlagCheckedTime = new Date().getTime(); @@ -262,8 +257,7 @@ public boolean isChannelPresent(String channelName) { //logger.info("Checking channelName: " + channelName + ", result = " + subscribedChannels.containsKey(channelName) + ", with message count = " + subscribedChannels.get(channelName).getCurrentMsgCount()); return (subscribedChannels != null) ? subscribedChannels.containsKey(channelName) : false; } catch (Exception e) { - logger.error(Thread.currentThread().getName() + ":: Unable to check if channel present: " + channelName); - e.printStackTrace(); + logger.error(Thread.currentThread().getName() + ":: Unable to check if channel present: " + channelName, e); return false; } } @@ -285,8 +279,7 @@ public void createChannelQueue(final String channelName) { subscribedChannels.put(channelName, cb); logger.info("Created channel buffer for channel: " + channelName + ", public = " + cb.getPubliclyListed()); } catch (Exception e) { - e.printStackTrace(); - logger.error("Unable to create buffer for channel: " + channelName); + logger.error("Unable to create buffer for channel: " + channelName, e); } } @@ -325,8 +318,7 @@ public Set getActiveChannelsList() { Set channelSet = (subscribedChannels != null && !subscribedChannels.isEmpty()) ? subscribedChannels.keySet() : null; return channelSet; } catch (Exception e) { - logger.error("Unable to fetch list of active channels"); - e.printStackTrace(); + logger.error("Unable to fetch list of active channels", e); return null; } } @@ -378,7 +370,6 @@ public Boolean getChannelPublicStatus(String channelName) { } } catch (Exception e) { logger.error("Error in querying manager for running collections: " + clientResponse); - logger.error(elog.toStringException(e)); } return true; // Question: should default be true or false? } @@ -409,7 +400,6 @@ public Map getAllRunningCollections() { } } catch (Exception e) { logger.error("Error in querying manager for running collections: " + clientResponse); - logger.error(elog.toStringException(e)); } return null; } @@ -464,9 +454,8 @@ public void run() { // Execute the blocking REDIS subscription call subscriberJedis.psubscribe(aidrSubscriber, channelRegEx); } catch (JedisConnectionException e) { - logger.error("AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx); + logger.error("AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx, e); System.out.println("[subscribeToChannel] AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx); - e.printStackTrace(); stopSubscription(); Thread.currentThread().interrupt(); } /*finally { @@ -492,7 +481,6 @@ private void stopSubscription() { } } catch (JedisConnectionException e) { logger.error("Connection to REDIS seems to be lost!"); - logger.error(elog.toStringException(e)); } try { if (jedisConn != null && aidrSubscriber != null && subscriberJedis != null) { @@ -503,7 +491,6 @@ private void stopSubscription() { } } catch (Exception e) { logger.error("Failed to return Jedis resource"); - logger.error(elog.toStringException(e)); } logger.info("isShutDown initiated = " + shutdownFlag); if (!shutdownFlag) { @@ -530,7 +517,6 @@ private void attemptResubscription() { isSubscribed = false; isSetup = false; logger.error("Fatal exception occurred attempting subscription: " + e.toString()); - logger.error(elog.toStringException(e)); ++attempts; } } @@ -545,7 +531,6 @@ private boolean setupRedisConnection(final String channelRegEx) { subscriberJedis = null; isConnected = false; logger.error("Fatal error! Couldn't establish connection to REDIS!"); - logger.error(elog.toStringException(e)); } if (isConnected) { aidrSubscriber = new RedisSubscriber(); @@ -560,7 +545,6 @@ private boolean setupRedisConnection(final String channelRegEx) { } catch (Exception e) { isSubscribed = false; logger.error("Fatal exception occurred attempting subscription: " + e.toString()); - logger.error(elog.toStringException(e)); } } return false; @@ -609,6 +593,7 @@ void shutdownAndAwaitTermination() { } } catch (InterruptedException e) { // (Re-)Cancel if current thread also interrupted + logger.error("Error in cleanup.", e); executorServicePool.shutdownNow(); // Preserve interrupt status Thread.currentThread().interrupt(); @@ -650,7 +635,6 @@ public void onPMessage(String pattern, String channel, String message) { } } catch (Exception e) { logger.error("Exception occurred, redisLoadShedder = " + redisLoadShedder + ", channel status: " + redisLoadShedder.containsKey(channel)); - logger.error(elog.toStringException(e)); } } diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/GetBufferedAIDRData.java b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/GetBufferedAIDRData.java index 80f9f540c..42f59ae7f 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/GetBufferedAIDRData.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/GetBufferedAIDRData.java @@ -35,12 +35,12 @@ import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.OPTIONS; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; -import javax.ws.rs.DefaultValue; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; @@ -48,15 +48,14 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.filter.ClassifiedFilteredTweet; +import qa.qcri.aidr.common.filter.DeserializeFilters; import qa.qcri.aidr.common.filter.FilterQueryMatcher; import qa.qcri.aidr.common.filter.JsonQueryList; import qa.qcri.aidr.output.utils.JsonDataFormatter; import qa.qcri.aidr.output.utils.OutputConfigurationProperty; import qa.qcri.aidr.output.utils.OutputConfigurator; import qa.qcri.aidr.output.utils.SimpleFairScheduler; -import qa.qcri.aidr.common.filter.DeserializeFilters; @Path("/crisis/fetch/") public class GetBufferedAIDRData implements ServletContextListener { @@ -220,7 +219,7 @@ public Response getBufferedAIDRData(@PathParam("crisisCode") String channelCode, } if (error) { - logger.warn("Error in requested channel name: " + channelCode); + logger.error("Error in requested channel name: " + channelCode); return Response.ok(new String("[{}]")).build(); } else { diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java b/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java index ce0431014..d6e9e68a7 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java @@ -32,12 +32,9 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import javax.servlet.ServletContextEvent; @@ -50,19 +47,16 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; -import org.glassfish.jersey.server.ChunkedOutput; import org.apache.log4j.Logger; +import org.glassfish.jersey.server.ChunkedOutput; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; - -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.output.utils.JedisConnectionObject; import qa.qcri.aidr.output.utils.OutputConfigurationProperty; import qa.qcri.aidr.output.utils.OutputConfigurator; -import qa.qcri.aidr.output.stream.AsyncStreamRedisSubscriber; import redis.clients.jedis.Jedis; +import com.google.gson.JsonObject; + @Path("/crisis/stream/") public class AsyncStream implements ServletContextListener { @@ -85,7 +79,6 @@ public class AsyncStream implements ServletContextListener { // Debugging private static Logger logger = Logger.getLogger(AsyncStream.class.getName()); - private static ErrorLog elog = new ErrorLog(); ///////////////////////////////////////////////////////////////////////////// @Override @@ -108,19 +101,16 @@ public void contextDestroyed(ServletContextEvent sce) { w.close(); } catch (IOException e) { logger.error("Error trying to close ChunkedOutput writer"); - logger.error(elog.toStringException(e)); } finally { try { w.close(); } catch (IOException e) { logger.error("Error trying to close ChunkedOutput writer"); - logger.error(elog.toStringException(e)); } } } } writerList.clear(); - logger.info("executorService: " + executorService); if (executorService != null) shutdownAndAwaitTermination(executorService); logger.info("Context destroyed"); } @@ -167,13 +157,11 @@ public void run() { sub.stopSubscription(jedisConn, subData); logger.error(subData.redisChannel + ": AIDR Predict Channel Subscribing failed"); - logger.error(elog.toStringException(e)); } finally { try { sub.stopSubscription(jedisConn, subData); } catch (Exception e) { logger.error(subData.redisChannel + ": Exception occurred attempting stopSubscription: " + e.toString()); - logger.error(elog.toStringException(e)); //System.exit(1); } } @@ -181,7 +169,6 @@ public void run() { }); } catch (RejectedExecutionException|NullPointerException e) { logger.error(subData.redisChannel + ": Fatal error executing async thread! Terminating."); - logger.error(elog.toStringException(e)); } } @@ -230,7 +217,6 @@ public ChunkedOutput streamChunkedResponse( } catch (Exception e) { // TODO Auto-generated catch block logger.error(channelCode + ": Fatal exception occurred attempting subscription: " + e.toString()); - logger.error(elog.toStringException(e)); //System.exit(1); } logger.info(channelCode + ": Spawning async response thread"); @@ -238,7 +224,6 @@ public ChunkedOutput streamChunkedResponse( executorService.execute(aidrSubscriber); } catch (RejectedExecutionException|NullPointerException e) { logger.error(channelCode + "Fatal error executing async thread! Terminating."); - logger.error(elog.toStringException(e)); } } } @@ -278,6 +263,8 @@ void shutdownAndAwaitTermination(ExecutorService threadPool) { logger.error("Executor Thread Pool did not terminate"); } } catch (InterruptedException ie) { + + logger.error("Error in clean up of threads.", ie); // (Re-)Cancel if current thread also interrupted threadPool.shutdownNow(); // Preserve interrupt status diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStreamRedisSubscriber.java b/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStreamRedisSubscriber.java index fbce689a8..554ea1a45 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStreamRedisSubscriber.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStreamRedisSubscriber.java @@ -21,8 +21,6 @@ import org.apache.log4j.Logger; import org.glassfish.jersey.server.ChunkedOutput; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.output.stream.SubscriptionDataObject; import qa.qcri.aidr.output.utils.JedisConnectionObject; import qa.qcri.aidr.output.utils.JsonDataFormatter; import redis.clients.jedis.Jedis; @@ -63,7 +61,6 @@ public class AsyncStreamRedisSubscriber extends JedisPubSub implements AsyncList // Debugging private static Logger logger = Logger.getLogger(AsyncStreamRedisSubscriber.class); - private static ErrorLog elog = new ErrorLog(); public AsyncStreamRedisSubscriber(final Jedis jedis, final ChunkedOutput responseWriter, ArrayList> writerList, @@ -121,8 +118,7 @@ public void onMessage(String channel, String message) { } } } catch (Exception e) { - logger.error("Error in onPMessage"); - logger.error(elog.toStringException(e)); + logger.error("Error in onPMessage channel : " + channel + " message : " + message); } } @@ -135,8 +131,7 @@ public void onPMessage(String pattern, String channel, String message) { } } } catch (Exception e) { - logger.error("Error in onPMessage"); - logger.error(elog.toStringException(e)); + logger.error("Error in onPMessage pattern : " + pattern + " channel : " + channel + " message : " + message); } } @@ -237,7 +232,7 @@ public void run() { } } catch (Exception e) { setRunFlag(false); - logger.warn(channel + ": Error in write attempt - possible client disconnect"); + logger.error(channel + ": Error in write attempt - possible client disconnect"); } if (count != 0) { // we did not just send an empty JSONP message lastAccessedTime = new Date().getTime(); // approx. time when message last received from REDIS @@ -252,6 +247,7 @@ public void run() { try { Thread.sleep(sleepTime); } catch (InterruptedException e) { + logger.warn("Error in sleep."); // TODO Auto-generated catch block //e.printStackTrace(); } @@ -273,6 +269,7 @@ public void run() { try { Thread.sleep(10); } catch (InterruptedException e) { + logger.warn("error in sleep."); // TODO Auto-generated catch block } } @@ -297,7 +294,6 @@ public void run() { writerList.remove(responseWriter); } catch (IOException e) { logger.error(channel + ": Error attempting closing ChunkedOutput."); - logger.error(elog.toStringException(e)); } } try { @@ -305,7 +301,6 @@ public void run() { } catch (Exception e) { // TODO Auto-generated catch block logger.error(channel + ": Attempting clean-up. Exception occurred attempting stopSubscription: " + e.toString()); - logger.error(elog.toStringException(e)); } } } diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/AIDROutputPing.java b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/AIDROutputPing.java index 86e6fe87b..dff3fd982 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/AIDROutputPing.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/AIDROutputPing.java @@ -150,8 +150,7 @@ public Response pingAIDROutput(@QueryParam("callback") String callbackName) { jedisConn.returnJedis(jedis); } } catch (JedisConnectionException e) { - logger.error("Error! Couldn't establish connection to REDIS!"); - logger.error(e); + logger.error("Error! Couldn't establish connection to REDIS!", e); } StringBuilder jsonpRes = new StringBuilder(); if (callbackName != null) jsonpRes.append(callbackName).append("("); diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JedisConnectionObject.java b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JedisConnectionObject.java index 2f418e1f0..09430c598 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JedisConnectionObject.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JedisConnectionObject.java @@ -7,12 +7,8 @@ import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; - - - import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @@ -36,7 +32,6 @@ public class JedisConnectionObject { // Logger setup private static Logger logger = Logger.getLogger(JedisConnectionObject.class); - private static ErrorLog elog = new ErrorLog(); /** * * @param host hostname on which REDIS resides @@ -78,7 +73,6 @@ public JedisConnectionObject(final String host, final int port) { logger.info("New Jedis pool: " + pool); } catch (Exception e) { logger.error("Fatal error! Could not initialize Jedis Pool!"); - logger.error(elog.toStringException(e)); JedisConnectionObject.poolConfig = null; JedisConnectionObject.pool = null; JedisConnectionObject.poolSetup = false; @@ -120,7 +114,6 @@ public Jedis getJedisResource() { subscriberJedis = null; connectionSetup = false; logger.error("Fatal error! Could not get a resource from the pool."); - logger.error(elog.toStringException(e)); } if (subscriberJedis != null) { allotedJedis.put(subscriberJedis, false); // initially nothing assigned @@ -171,7 +164,6 @@ public void returnJedis(Jedis jedisInstance) { } } catch (JedisConnectionException e) { logger.error("JedisConnectionException occurred..."); - logger.error(elog.toStringException(e)); if (jedisInstance != null && jedisInstance.isConnected()) jedisInstance.close(); jedisInstance = null; //JedisConnectionObject.pool.returnBrokenResource(jedisInstance); diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JsonDataFormatter.java b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JsonDataFormatter.java index 88a08ed26..e3f51850e 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JsonDataFormatter.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JsonDataFormatter.java @@ -4,10 +4,8 @@ package qa.qcri.aidr.output.utils; -import java.util.HashMap; import java.util.List; import java.util.ListIterator; -import java.util.Map; import org.apache.log4j.Logger; diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/TaggerJsonOutputAdapter.java b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/TaggerJsonOutputAdapter.java index 9e3bc5dd2..269f2938d 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/TaggerJsonOutputAdapter.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/TaggerJsonOutputAdapter.java @@ -19,8 +19,6 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import qa.qcri.aidr.common.logging.ErrorLog; - public class TaggerJsonOutputAdapter { // Logger setup private static Logger logger = Logger.getLogger(TaggerJsonOutputAdapter.class); @@ -131,9 +129,7 @@ public String buildJsonString(String rawJsonString, boolean rejectNullFlag) { } } } catch (Exception e) { - logger.error("Exception in json parsing for string: " + rawJsonString); - logger.error("exception", e); - e.printStackTrace(); + logger.error("Exception in json parsing for string: " + rawJsonString, e); } // no group label called "aidr" or "text" present diff --git a/aidr_testers.sh b/aidr_testers.sh index 6594665b9..a315131e2 100644 --- a/aidr_testers.sh +++ b/aidr_testers.sh @@ -38,7 +38,7 @@ fi #Tagger Tester cd $AIDR_HOME cd aidr-tagger -mvn -P$PROFILE test -Dtest=TaggerTesterTest -Dnitems-test=100 +mvn -P$PROFILE test -Dtest=TaggerTesterTest -DskipTests=false -Dnitems-test=100 result=$? #result=0(success), 1(fail) if [[ $result != 0 ]] ; then From 02b49174964d41fec4bb3028ad969492641e1693 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Tue, 28 Jul 2015 16:05:43 +0300 Subject: [PATCH 10/44] updated code to address Latika's issue - needs testing by Latika; 2 new REST APIs --- .../aidr/analysis/api/ApplicationConfig.java | 2 +- .../analysis/api/GetTagDataStatistics.java | 55 +++-- .../analysis/api/WriteStatisticsData.java | 2 + .../TagDataStatisticsResourceFacade.java | 7 + .../TagDataStatisticsResourceFacadeImp.java | 190 +++++++++++++----- .../service/GetTagDataStatisticsService.java | 69 ++++++- .../aidr/analysis/stat/TagDataMapRecord.java | 9 + .../aidr/analysis/utils/ChannelBuffer.java | 164 --------------- .../analysis/utils/ChannelBufferManager.java | 18 +- .../aidr/analysis/utils/JsonResponse.java | 4 +- .../src/main/webapp/WEB-INF/web.xml | 2 +- 11 files changed, 285 insertions(+), 237 deletions(-) delete mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBuffer.java diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java index ccb24d782..2db2908ea 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/ApplicationConfig.java @@ -14,7 +14,7 @@ * * @author Imran */ -@javax.ws.rs.ApplicationPath("rest") +@javax.ws.rs.ApplicationPath("/rest/analytics/") public class ApplicationConfig extends Application { @Override diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java index b82f4c862..95daa3ce1 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java @@ -2,6 +2,7 @@ package qa.qcri.aidr.analysis.api; +import javax.ejb.EJB; import javax.json.Json; import javax.json.JsonObject; import javax.servlet.ServletContextEvent; @@ -19,19 +20,48 @@ import org.apache.log4j.Logger; - +import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; import qa.qcri.aidr.analysis.service.GetTagDataStatisticsService; /** * This is the REST API interface for accessing the aidr_analytics DB's tag_data entity. */ -@Path("/tagData/") +@Path("/tagdata/") public class GetTagDataStatistics implements ServletContextListener { // Debugging private static Logger logger = Logger.getLogger(GetTagDataStatistics.class); + @EJB + private GetTagDataStatisticsService tagDataService; + + + @GET + @Path("/getlabelsum/{crisisCode}/{granularity}") + @Produces(MediaType.APPLICATION_JSON) + public Response getTagCountSumForAllAttributesFromTime(@PathParam("crisisCode") String crisisCode, + @PathParam("granularity") Long granularity, + @DefaultValue("0") @QueryParam("startTime") Long startTime) { + + System.out.println("local EJB in REST API = " + tagDataService); + JSONObject json = tagDataService.getTagCountSumForAllAttributesFromTime(crisisCode, granularity, startTime); + return Response.ok(json.toJSONString()).build(); + } + + + @GET + @Path("/getlabelsum/{crisisCode}") + @Produces(MediaType.APPLICATION_JSON) + public Response getTagCountSumForAllGranularitiesFromTime(@PathParam("crisisCode") String crisisCode, + @DefaultValue("0") @QueryParam("startTime") Long startTime) { + + System.out.println("local EJB in REST API = " + tagDataService); + JSONObject json = tagDataService.getTagCountSumByGranularity(crisisCode, startTime); + return Response.ok(json.toJSONString()).build(); + } + + /** * * @param crisisCode @@ -41,14 +71,15 @@ public class GetTagDataStatistics implements ServletContextListener { * @return Count sum per label from startTime to current time at the given granularity, for crisisCode and attributeCode */ @GET - @Path("/getLabelSum/{crisisCode}/{classifierCode}/{granularity}") + @Path("/getlabelsum/{crisisCode}/{attributeCode}/{granularity}") @Produces(MediaType.APPLICATION_JSON) public Response getTagCountSumFromTime(@PathParam("crisisCode") String crisisCode, @PathParam("attributeCode") String attributeCode, @PathParam("granularity") Long granularity, @DefaultValue("0") @QueryParam("startTime") Long startTime) { - JSONObject json = new GetTagDataStatisticsService().getTagCountSumFromTime(crisisCode, attributeCode, granularity, startTime); + System.out.println("local EJB in REST API = " + tagDataService); + JSONObject json = tagDataService.getTagCountSumFromTime(crisisCode, attributeCode, granularity, startTime); return Response.ok(json.toJSONString()).build(); } @@ -61,14 +92,14 @@ public Response getTagCountSumFromTime(@PathParam("crisisCode") String crisisCod * @return Count per label in the specified time window at the given granularity, for crisisCode and attributeCode */ @GET - @Path("/getLabelCount/{crisisCode}/{attributeCode}/{granularity}") + @Path("/getlabelcount/{crisisCode}/{attributeCode}/{granularity}") @Produces(MediaType.APPLICATION_JSON) public Response getTagCountInTimeWindow(@PathParam("crisisCode") String crisisCode, @PathParam("attributeCode") String attributeCode, @PathParam("granularity") Long granularity, @DefaultValue("0") @QueryParam("startTime") Long startTime) { - JSONObject json = new GetTagDataStatisticsService().getTagCountInTimeWindow(crisisCode, attributeCode, granularity, startTime); + JSONObject json = tagDataService.getTagCountInTimeWindow(crisisCode, attributeCode, granularity, startTime); return Response.ok(json.toJSONString()).build(); } @@ -82,7 +113,7 @@ public Response getTagCountInTimeWindow(@PathParam("crisisCode") String crisisCo * @return Time series data for each label in the interval [startTime, endTime] at the given granularity, for crisisCode and attributeCode */ @GET - @Path("/getLabelTimeSeries/{crisisCode}/{attributeCode}/{granularity}") + @Path("/getlabeltimeseries/{crisisCode}/{attributeCode}/{granularity}") @Produces(MediaType.APPLICATION_JSON) public Response getTagCountTimeSeries(@PathParam("crisisCode") String crisisCode, @PathParam("attributeCode") String attributeCode, @@ -93,7 +124,7 @@ public Response getTagCountTimeSeries(@PathParam("crisisCode") String crisisCode if (null == endTime || endTime < startTime) { endTime = System.currentTimeMillis(); } - JSONObject json = new GetTagDataStatisticsService().getTagCountTimeSeries(crisisCode, attributeCode, granularity, startTime, endTime); + JSONObject json = tagDataService.getTagCountTimeSeries(crisisCode, attributeCode, granularity, startTime, endTime); return Response.ok(json.toJSONString()).build(); } @@ -107,7 +138,7 @@ public Response getTagCountTimeSeries(@PathParam("crisisCode") String crisisCode * @return Count sum for each label in the interval [startTime, endTime] at the given granularity, for crisisCode and attributeCode */ @GET - @Path("/getIntervalLabelSum/{crisisCode}/{attributeCode}/{granularity}") + @Path("/getintervallabelsum/{crisisCode}/{attributeCode}/{granularity}") @Produces(MediaType.APPLICATION_JSON) public Response getTagCountSumInInterval(@PathParam("crisisCode") String crisisCode, @PathParam("attributeCode") String attributeCode, @@ -115,7 +146,7 @@ public Response getTagCountSumInInterval(@PathParam("crisisCode") String crisisC @DefaultValue("0") @QueryParam("startTime") Long startTime, @QueryParam("endTime") Long endTime) { - JSONObject json = new GetTagDataStatisticsService().getTagCountSumInInterval(crisisCode, attributeCode, granularity, startTime, endTime); + JSONObject json = tagDataService.getTagCountSumInInterval(crisisCode, attributeCode, granularity, startTime, endTime); return Response.ok(json.toJSONString()).build(); } @@ -130,14 +161,14 @@ public Response getTagCountSumInInterval(@PathParam("crisisCode") String crisisC * @return The count for a label in the specified time window at the given granularity, for crisisCode and attributeCode */ @GET - @Path("/getOneLabelData/{crisisCode}/{attributeCode}/{labelCode}/{granularity}/{startTime}") + @Path("/getonelabeldata/{crisisCode}/{attributeCode}/{labelCode}/{granularity}/{startTime}") @Produces(MediaType.APPLICATION_JSON) public Response getSingleItem(@PathParam("crisisCode") String crisisCode, @PathParam("attributeCode") String attributeCode, @PathParam("labelCode") String labelCode, @PathParam("granularity") Long granularity, @PathParam("startTime") Long startTime) { - JSONObject json = new GetTagDataStatisticsService().getSingleItem(crisisCode, attributeCode, labelCode, granularity, startTime); + JSONObject json = tagDataService.getSingleItem(crisisCode, attributeCode, labelCode, granularity, startTime); return Response.ok(json.toJSONString()).build(); } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java index a01cb1f32..9aebeedcf 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -140,6 +140,8 @@ private ReturnCode writeOutputDataToTagDataDB(Long granularity,Long timestamp) { TagData t = new TagData(key.getCrisisCode(), timestamp, granularity, key.getAttributeCode(), key.getLabelCode(), tCount.getCount(granularity)); if (tCount.getCount(granularity) > 0) { System.out.println("Will attempt persistence of tag key: " + key.toString()); + t.setMaxCreatedAt(tCount.getLastUpdateTime()); + t.setMinCreatedAt(tCount.getFirstEntryTime()); tagDataEJB.writeData(t); } tCount.resetCount(granularity); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java index b741ba1f6..656e0e36e 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/TagDataStatisticsResourceFacade.java @@ -2,6 +2,7 @@ package qa.qcri.aidr.analysis.facade; import java.util.List; +import java.util.Map; import javax.ejb.Local; import javax.persistence.EntityManager; @@ -49,4 +50,10 @@ public List getDataInInterval(String crisisCode, String attributeCode, public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2, Long granularity); + + public List getAttributesForCrisis(String crisisCode); + + public List getDataByCrisisGranularity(String crisisCode, Long timestamp, Long granularity); + public Map getTagCountByCrisisGranularity(String crisisCode, Long timestamp); + public List getGranularitiesForCrisis(String crisisCode); } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java index f0627ece6..3e964e63b 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java @@ -1,7 +1,9 @@ package qa.qcri.aidr.analysis.facade.imp; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.ejb.Stateless; import javax.persistence.EntityManager; @@ -12,12 +14,12 @@ import org.hibernate.HibernateException; import org.hibernate.Session; import org.hibernate.criterion.Criterion; +import org.hibernate.criterion.Projections; import org.hibernate.criterion.Restrictions; import qa.qcri.aidr.analysis.entity.TagData; import qa.qcri.aidr.analysis.entity.TagDataPK; import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; - import qa.qcri.aidr.common.values.ReturnCode; /** @@ -27,7 +29,7 @@ public class TagDataStatisticsResourceFacadeImp implements TagDataStatisticsResourceFacade { private static Logger logger = Logger.getLogger(TagDataStatisticsResourceFacadeImp.class); - + @PersistenceContext(unitName = "qa.qcri.aidr.analysis-EJBS") private EntityManager em; @@ -67,8 +69,8 @@ public Session getCurrentSession() { throw new HibernateException("getCurrentSession failed"); } } - - + + @Override public ReturnCode writeData(TagData tagData) { try { @@ -105,13 +107,107 @@ public TagData getSingleDataByPK(TagDataPK tagDataPK) { } return null; } - + @SuppressWarnings("unchecked") @Override public List getDataByCrisis(String crisisCode) { Criteria criteria = getCurrentSession().createCriteria(TagData.class); criteria.add(Restrictions.eq("crisisCode", crisisCode)); System.out.println("Formed criteria: " + criteria.toString()); + try { + List objList = (List) criteria.list(); + //System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); + return objList; + } catch (HibernateException e) { + //logger.error(elog.toStringException(e)); + e.printStackTrace(); + } + return null; + } + + // TODO + @SuppressWarnings("unchecked") + @Override + public Map getTagCountByCrisisGranularity(String crisisCode, Long timestamp) { + try { + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.ge("timestamp", timestamp)); + + List countList = getCurrentSession().createCriteria(TagData.class) + .add(criterion) + .setProjection(Projections.distinct(Projections.projectionList() + .add(Projections.groupProperty("granularity").as("granularity")) + .add(Projections.sum("count").as("count"))) + ).list(); + + //System.out.println("fetched list size = " + (countList != null ? countList.size() : "null")); + Map data = new HashMap(); + for (int i = 0; i < countList.size();i++) { + Object[] temp = (Object[]) countList.get(i); + Long g = ((Number) temp[0]).longValue(); + Long count = ((Number) temp[1]).longValue(); + data.put(g.toString(), count); + //System.out.println("Fetched {" + temp[0] + ", " + temp[1] + "}"); + //System.out.println("1 --> " + g.getClass() + ", 2 --> " + temp[1].getClass()); + + } + return data; + } catch (HibernateException e) { + //logger.error("exception: ", e); + e.printStackTrace(); + } + return null; + } + + + @SuppressWarnings("unchecked") + @Override + public List getAttributesForCrisis(String crisisCode) { + try { + List attributesList = (List) getCurrentSession().createCriteria(TagData.class) + .add(Restrictions.eq("crisisCode", crisisCode)) + .setProjection(Projections.distinct(Projections.property("attributeCode"))).list(); + //System.out.println("fetched attributes list size = " + (attributesList != null ? attributesList.size() : "null")); + return attributesList; + } catch (Exception e) { + e.printStackTrace(); + logger.error("exception: ", e); + return null; + } + } + + @SuppressWarnings("unchecked") + @Override + public List getGranularitiesForCrisis(String crisisCode) { + try { + List gList = (List) getCurrentSession().createCriteria(TagData.class) + .add(Restrictions.eq("crisisCode", crisisCode)) + .setProjection(Projections.distinct(Projections.property("granularity"))).list(); + System.out.println("fetched attributes list size = " + (gList != null ? gList.size() : "null")); + return gList; + } catch (Exception e) { + e.printStackTrace(); + logger.error("exception: ", e); + return null; + } + + } + + @SuppressWarnings("unchecked") + @Override + public List getDataByCrisisGranularity(String crisisCode, Long timestamp, Long granularity) { + Criteria criteria = getCurrentSession().createCriteria(TagData.class); + Criterion criterion = Restrictions.conjunction() + .add(Restrictions.eq("crisisCode", crisisCode)) + .add(Restrictions.ge("timestamp", timestamp)); + if (granularity != null) { + criterion = Restrictions.conjunction() + .add(criterion) + .add(Restrictions.eq("granularity", granularity)); + } + criteria.add(criterion); + System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); @@ -122,6 +218,8 @@ public List getDataByCrisis(String crisisCode) { } return null; } + + @SuppressWarnings("unchecked") @Override @@ -132,8 +230,8 @@ public List getDataByCrisisAttributeLabel(String crisisCode, String att .add(Restrictions.eq("attributeCode", attributeCode)); if (labelCode != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("labelCode", labelCode)); + .add(criterion) + .add(Restrictions.eq("labelCode", labelCode)); } criteria.add(criterion); System.out.println("Formed criteria: " + criteria.toString()); @@ -147,7 +245,7 @@ public List getDataByCrisisAttributeLabel(String crisisCode, String att } return null; } - + @Override public List getDataByCrisisAttributeLabelGranularity(String crisisCode, String attributeCode, String labelCode, Long granularity) { Criteria criteria = getCurrentSession().createCriteria(TagData.class); @@ -168,9 +266,8 @@ public List getDataByCrisisAttributeLabelGranularity(String crisisCode, } return null; } - - public List getDataByGranularityInTimeWindow(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity) { + + public List getDataByGranularityInTimeWindow(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity) { Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) @@ -179,10 +276,10 @@ public List getDataByGranularityInTimeWindow(String crisisCode, String .add(Restrictions.eq("timestamp", timestamp)); if (labelCode != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("labelCode", labelCode)); + .add(criterion) + .add(Restrictions.eq("labelCode", labelCode)); } - + criteria.add(criterion); System.out.println("Formed criteria: " + criteria.toString()); try { @@ -195,7 +292,7 @@ public List getDataByGranularityInTimeWindow(String crisisCode, String } return null; } - + @Override public List getDataAfterTimestamp(String crisisCode, String attributeCode, String labelCode, Long timestamp) { Criteria criteria = getCurrentSession().createCriteria(TagData.class); @@ -205,10 +302,10 @@ public List getDataAfterTimestamp(String crisisCode, String attributeCo .add(Restrictions.ge("timestamp", timestamp)); if (labelCode != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("labelCode", labelCode)); + .add(criterion) + .add(Restrictions.eq("labelCode", labelCode)); } - + criteria.add(criterion); System.out.println("Formed criteria: " + criteria.toString()); try { @@ -221,10 +318,9 @@ public List getDataAfterTimestamp(String crisisCode, String attributeCo } return null; } - + @Override - public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity) { + public List getDataAfterTimestampGranularity(String crisisCode, String attributeCode, String labelCode, Long timestamp, Long granularity) { Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) @@ -233,8 +329,8 @@ public List getDataAfterTimestampGranularity(String crisisCode, String .add(Restrictions.eq("granularity", granularity)); if (labelCode != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("labelCode", labelCode)); + .add(criterion) + .add(Restrictions.eq("labelCode", labelCode)); } criteria.add(criterion); System.out.println("Formed criteria: " + criteria.toString()); @@ -258,10 +354,10 @@ public List getDataBeforeTimestamp(String crisisCode, String attributeC .add(Restrictions.le("timestamp", timestamp)); if (labelCode != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("labelCode", labelCode)); + .add(criterion) + .add(Restrictions.eq("labelCode", labelCode)); } - + criteria.add(criterion); System.out.println("Formed criteria: " + criteria.toString()); try { @@ -274,10 +370,10 @@ public List getDataBeforeTimestamp(String crisisCode, String attributeC } return null; } - + @Override public List getDataBeforeTimestampGranularity(String crisisCode, String attributeCode, String labelCode, - Long timestamp, Long granularity) { + Long timestamp, Long granularity) { Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.conjunction() .add(Restrictions.eq("crisisCode", crisisCode)) @@ -286,10 +382,10 @@ public List getDataBeforeTimestampGranularity(String crisisCode, String .add(Restrictions.eq("granularity", granularity)); if (labelCode != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("labelCode", labelCode)); + .add(criterion) + .add(Restrictions.eq("labelCode", labelCode)); } - + criteria.add(criterion); System.out.println("Formed criteria: " + criteria.toString()); try { @@ -303,7 +399,7 @@ public List getDataBeforeTimestampGranularity(String crisisCode, String return null; } - + @Override public List getDataInInterval(String crisisCode, String attributeCode, String labelCode, Long timestamp1, Long timestamp2) { Criteria criteria = getCurrentSession().createCriteria(TagData.class); @@ -314,8 +410,8 @@ public List getDataInInterval(String crisisCode, String attributeCode, .add(Restrictions.le("timestamp", timestamp2)); if (labelCode != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("labelCode", labelCode)); + .add(criterion) + .add(Restrictions.eq("labelCode", labelCode)); } criteria.add(criterion); System.out.println("Formed criteria: " + criteria.toString()); @@ -329,38 +425,38 @@ public List getDataInInterval(String crisisCode, String attributeCode, } return null; } - + @Override public List getDataInIntervalWithGranularity(String crisisCode, String attributeCode, String labelCode, - Long timestamp1, Long timestamp2, Long granularity) { + Long timestamp1, Long timestamp2, Long granularity) { Criteria criteria = getCurrentSession().createCriteria(TagData.class); Criterion criterion = Restrictions.eq("crisisCode", crisisCode); - + // Now add the optional non-null criteria if (attributeCode != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("attributeCode", attributeCode)); + .add(criterion) + .add(Restrictions.eq("attributeCode", attributeCode)); } if (labelCode != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("labelCode", labelCode)); + .add(criterion) + .add(Restrictions.eq("labelCode", labelCode)); } if (granularity != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.eq("granularity", granularity)); + .add(criterion) + .add(Restrictions.eq("granularity", granularity)); } if (timestamp1 != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.ge("timestamp", timestamp1)); + .add(criterion) + .add(Restrictions.ge("timestamp", timestamp1)); } if (timestamp2 != null) { criterion = Restrictions.conjunction() - .add(criterion) - .add(Restrictions.le("timestamp", timestamp2)); + .add(criterion) + .add(Restrictions.le("timestamp", timestamp2)); } criteria.add(criterion); System.out.println("Formed criteria: " + criteria.toString()); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java index 39fcd73f4..66eccf403 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java @@ -2,11 +2,16 @@ package qa.qcri.aidr.analysis.service; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; +import javax.annotation.PostConstruct; import javax.ejb.EJB; +import javax.ejb.Local; +import javax.ejb.Stateless; import net.minidev.json.JSONObject; @@ -26,6 +31,8 @@ /** * Implements the business logic for providing the REST API functionalities on tag_data table. */ + +@Stateless public class GetTagDataStatisticsService { // Debugging @@ -34,6 +41,63 @@ public class GetTagDataStatisticsService { @EJB private TagDataStatisticsResourceFacade tagDataEJB; + public GetTagDataStatisticsService() {} + + public JSONObject getTagCountSumForAllAttributesFromTime(String crisisCode, Long granularity, Long startTime) { + List attributesList = tagDataEJB.getAttributesForCrisis(crisisCode); + List tagDataList = new ArrayList(); + JSONObject jsonList = new JSONObject(); + jsonList.put("crisisCode", crisisCode); + jsonList.put("granularity", granularity); + jsonList.put("startTime", startTime); + + if (attributesList != null) { + jsonList.put("attributesList", attributesList); + Map dataSet = new HashMap(); + for (String attribute: attributesList) { + System.out.println("For attribute: " + attribute); + tagDataList.addAll(tagDataEJB.getDataAfterTimestampGranularity(crisisCode, attribute, null, startTime, granularity)); + // Now the real work - count and send response + JSONObject json = new JsonResponse().getNewJsonResponseObject(crisisCode, attribute, granularity, startTime, null); + + if (tagDataList != null) { + Map tagCountMap = new TreeMap(); + for (TagData t: tagDataList) { + if (tagCountMap.containsKey(t.getLabelCode())) { + tagCountMap.put(t.getLabelCode(), tagCountMap.get(t.getLabelCode()) + t.getCount()); + } else { + tagCountMap.put(t.getLabelCode(), t.getCount()); + } + } + try { + json.put("data", tagCountMap); + dataSet.put(attribute, json); + System.out.println("Added json: " + json.toJSONString()); + } catch (Exception e) { + json = JsonResponse.addError(json); + logger.error("Error in serializing fetched tag count data", e); + } + } + } + jsonList.put("attribute_data", dataSet); + } + return jsonList; + } + + public JSONObject getTagCountSumByGranularity(String crisisCode, Long startTime) { + List gList = tagDataEJB.getGranularitiesForCrisis(crisisCode); + Map countList = tagDataEJB.getTagCountByCrisisGranularity(crisisCode, startTime); + JSONObject json = new JSONObject(); + json.put("crisisCode", crisisCode); + json.put("startTime", startTime); + if (gList != null) { + json.put("granularities", gList); + json.put("data", countList); + } + //System.out.println("Returned json: " + json.toJSONString()); + return json; + } + /** * * @param crisisCode @@ -44,7 +108,8 @@ public class GetTagDataStatisticsService { */ public JSONObject getTagCountSumFromTime(String crisisCode, String attributeCode, Long granularity, Long startTime) { - + System.out.println("Received parameters: crisisCode = " + crisisCode + ", attributeCode = " + attributeCode + ", granularity = " + granularity + + ", startTime = " + startTime + ", local EJB = " + tagDataEJB); // First get the list of data points from DB List tagDataList = tagDataEJB.getDataAfterTimestampGranularity(crisisCode, attributeCode, null, startTime, granularity); @@ -125,7 +190,7 @@ public JSONObject getTagCountTimeSeries(String crisisCode, String attributeCode, // Now the real work - creat time series, format and send response JSONObject json = new JsonResponse().getNewJsonResponseObject(crisisCode, attributeCode, granularity, startTime, endTime); - + if (tagDataList != null) { // Create time series Map data first Map> tagCountMap = new TreeMap>(); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java index 019c010a9..94bc6c987 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java @@ -7,7 +7,15 @@ public class TagDataMapRecord extends MapRecord { + private Long firstEntryTime; + public Long getFirstEntryTime() { + return firstEntryTime; + } + + public void setFirstEntryTime(Long firstEntryTime) { + this.firstEntryTime = firstEntryTime; + } public TagDataMapRecord(List granularityList) { this.count = new ConcurrentHashMap(); @@ -16,6 +24,7 @@ public TagDataMapRecord(List granularityList) { this.count.put(g, 0); } this.lastUpdateTime = System.currentTimeMillis(); + this.setFirstEntryTime(System.currentTimeMillis()); } public int getCount(Long g) { diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBuffer.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBuffer.java deleted file mode 100644 index e2635b6d6..000000000 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBuffer.java +++ /dev/null @@ -1,164 +0,0 @@ -/** - * @author Koushik Sinha - * Last modified: 06/01/2014 - * - * The ChannelBuffer class implements the creation, - * and deletion of channels as well as retrieval of - * messages from a specific channel. - * - */ - -package qa.qcri.aidr.analysis.utils; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import org.apache.commons.collections.Buffer; -import org.apache.commons.collections.BufferUtils; -import org.apache.commons.collections.buffer.CircularFifoBuffer; - - -import org.apache.log4j.Logger; - - - -public class ChannelBuffer { - public static int MAX_BUFFER_SIZE = 2000; // number of elements the buffer will hold at any time - public static int MAX_FETCH_SIZE = 2000; // max. number of elements to fetch in one op - private String channelName = null; - private long lastAddTime; - private Buffer messageBuffer = null; - int size = 0; - - private Boolean publiclyListed = true; - - private static Logger logger = Logger.getLogger(ChannelBuffer.class); - - public ChannelBuffer(final String name, final int bufferSize) { - this.channelName = name; - this.messageBuffer = BufferUtils.synchronizedBuffer(new CircularFifoBuffer(bufferSize)); - this.size = bufferSize; - } - - public ChannelBuffer(final String name) { - this.channelName = name; - } - - public void createChannelBuffer() { - this.messageBuffer = BufferUtils.synchronizedBuffer(new CircularFifoBuffer(MAX_BUFFER_SIZE)); - this.size = MAX_BUFFER_SIZE; - } - - public void createChannelBuffer(final int bufferSize) { - this.messageBuffer = BufferUtils.synchronizedBuffer(new CircularFifoBuffer(bufferSize)); - this.size = bufferSize; - } - - public void setPubliclyListed(Boolean publiclyListed) { - this.publiclyListed = publiclyListed; - } - - public Boolean getPubliclyListed() { - return publiclyListed; - } - - @SuppressWarnings("unchecked") - public void addMessage(String msg) { - try { - //synchronized(this.messageBuffer) - { - messageBuffer.add(msg); - lastAddTime = new Date().getTime(); - } - } catch (Exception e) { - logger.error("Couldn't add message to buffer for channel: " + this.channelName); - logger.error("exception", e); - } - } - - @SuppressWarnings("unchecked") - public void addAllMessages(ArrayList msgList) { - try { - //synchronized(this.messageBuffer) - { - messageBuffer.addAll(msgList); - lastAddTime = new Date().getTime(); - } - } catch (Exception e) { - logger.error("Couldn't add message to buffer for channel: " + this.channelName); - logger.error("exception", e); - } - } - - @SuppressWarnings("unchecked") - /** - * @param msgCount: number of messages to return - * @return Returns a list of messages sorted in ascending order of timestamp - */ - public List getMessages(int msgCount) { - //long startTime = System.currentTimeMillis(); - - List tempList = null; - try { - synchronized(this.messageBuffer) { - tempList = new ArrayList(this.messageBuffer.size()); - tempList.addAll(this.messageBuffer); - } - //logger.info("Copied data : " + tempList.size() + ", msgCount:messageBuffer.size = " + msgCount + ":" + messageBuffer.size()); - if (msgCount >= tempList.size()) { - return tempList; // optimization - } - - // Otherwise, get the last msgCount elements, in oldest-first order - // Collections.reverse(tempList); // in-situ reversal O(n) time - List returnList = new ArrayList(msgCount); - int index = Math.max(0, (tempList.size() - msgCount)); // from where to pick - for (int i = index;i < tempList.size();i++) { - returnList.add(tempList.get(i)); - } - //logger.info("Fetched size = " + msgCount + " from start loc = " + Math.max(0, (tempList.size() - msgCount))); - return returnList; - } catch (Exception e) { - logger.error("Error in creating list out of buffered messages"); - logger.error("exception", e); - return null; - } - } - - public int getCurrentMsgCount() { - return messageBuffer.size(); - } - - public void deleteBuffer() { - channelName = null; - if (messageBuffer != null) { - messageBuffer.clear(); - messageBuffer = null; - } - } - - public void setChannelName(String name) { - channelName = name; - } - - public String getChannelName() { - return channelName; - } - - public long getLastAddTime() { - return lastAddTime; - } - - public int getMaxBufferSize() { - return MAX_BUFFER_SIZE; - } - - public int getMaxFetchSize() { - return MAX_FETCH_SIZE; - } - - public int getBufferSize() { - return size; - } -} \ No newline at end of file diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java index c18a9bc83..c50b78de7 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java @@ -219,7 +219,7 @@ public void manageChannelBuffers(final String subscriptionPattern, final String ClassifiedFilteredTweet classifiedTweet = new ClassifiedFilteredTweet().deserialize(receivedMessage); if (classifiedTweet != null && classifiedTweet.getNominalLabels() != null && !classifiedTweet.getNominalLabels().isEmpty()) { channelMap.putIfAbsent(classifiedTweet.getCrisisCode(), System.currentTimeMillis()); - System.out.println("Found a valid classified tweet for collection: " + classifiedTweet.getCrisisCode() + ", tweet = " + receivedMessage); + //System.out.println("Found a valid classified tweet for collection: " + classifiedTweet.getCrisisCode() + ", tweet = " + receivedMessage); for (NominalLabel nb : classifiedTweet.getNominalLabels()) { if (nb.attribute_code != null && nb.label_code != null) { CounterKey tagDataKey = new CounterKey(classifiedTweet.getCrisisCode(), nb.attribute_code, nb.label_code); @@ -229,23 +229,24 @@ public void manageChannelBuffers(final String subscriptionPattern, final String TagDataMapRecord t = (TagDataMapRecord) tagDataMap.get(tagDataKey); t.incrementAllCounts(); tagDataMap.put(tagDataKey, t); - System.out.println("Updated Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); + //System.out.println("Updated Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); } else { TagDataMapRecord t = new TagDataMapRecord(granularityList); tagDataMap.put(tagDataKey, t); - logger.info("New Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); - System.out.println("New Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); + + //logger.info("New Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); + //System.out.println("New Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); } if (confDataMap.containsKey(confDataKey)) { ConfDataMapRecord f = (ConfDataMapRecord) confDataMap.get(confDataKey); f.incrementAllCounts(); confDataMap.put(confDataKey, f); - System.out.println("Updated Conf map entry with key: " + confDataKey.toString() + " value = " + confDataMap.get(confDataKey).toString()); + //System.out.println("Updated Conf map entry with key: " + confDataKey.toString() + " value = " + confDataMap.get(confDataKey).toString()); } else { ConfDataMapRecord t = new ConfDataMapRecord(granularityList); confDataMap.put(confDataKey, t); - logger.info("[manageChannelBuffersWrapper] New Conf map entry with key: " + confDataKey + " value = " + confDataMap.get(confDataKey)); - System.out.println("New Conf map entry with key: " + confDataKey.toString() + " value = " + confDataMap.get(confDataKey).toString()); + //logger.info("[manageChannelBuffersWrapper] New Conf map entry with key: " + confDataKey + " value = " + confDataMap.get(confDataKey)); + //System.out.println("New Conf map entry with key: " + confDataKey.toString() + " value = " + confDataMap.get(confDataKey).toString()); } } } @@ -266,7 +267,7 @@ private long periodicInactiveChannelCheck(long lastCheckedTime, ConcurrentHashMa if (currentTime - lastCheckedTime > ChannelBufferManager.CHECK_INTERVAL) { for (String key : channelMap.keySet()) { if ((currentTime - channelMap.get(key)) > ChannelBufferManager.NO_DATA_TIMEOUT) { - logger.info("Deleting data for inactive channel = " + key); + logger.info("Attempt deleting data for inactive channel = " + key); int deleteCount = deleteMapRecordsForCollection(key, dataMap); logger.info("Deleted records count for inactive channel <" + key + "> is = " + deleteCount); } @@ -509,6 +510,7 @@ public void onPMessage(String pattern, String channel, String message) { } if (redisLoadShedder.get(channel).canProcess(channel)) { manageChannelBuffers(pattern, channel, message); + System.out.println("Done putting message"); } } catch (Exception e) { logger.error("Exception occurred, redisLoadShedder = " + redisLoadShedder + ", channel status: " + redisLoadShedder.containsKey(channel)); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java index a17b0125d..a500f3f14 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java @@ -9,7 +9,7 @@ public class JsonResponse { JSONObject json = new JSONObject(); public JsonResponse() { - json.put("errorMsg", ReturnCode.SUCCESS); + json.put("status", ReturnCode.SUCCESS); } public JSONObject getNewJsonResponseObject(String crisisCode, String attributeCode, Long granularity, Long startTime, Long endTime) { @@ -32,7 +32,7 @@ public JSONObject getNewJsonResponseObject(String crisisCode, String attributeCo } public static JSONObject addError(JSONObject json) { - json.put("errorMsg", ReturnCode.ERROR); + json.put("status", ReturnCode.ERROR); return json; } } diff --git a/aidr-analytics/src/main/webapp/WEB-INF/web.xml b/aidr-analytics/src/main/webapp/WEB-INF/web.xml index 7674866ae..119a3926b 100644 --- a/aidr-analytics/src/main/webapp/WEB-INF/web.xml +++ b/aidr-analytics/src/main/webapp/WEB-INF/web.xml @@ -18,7 +18,7 @@ javax.ws.rs.core.Application - /rest/analytics/* + /rest/* From 396662ebd741fce891bae0a6b663c51778c76984 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Tue, 28 Jul 2015 16:10:00 +0300 Subject: [PATCH 11/44] updated documentation of 2 new REST APIs;reduced debug msgs --- .../aidr/analysis/api/GetTagDataStatistics.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java index 95daa3ce1..781499d32 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java @@ -27,6 +27,10 @@ * This is the REST API interface for accessing the aidr_analytics DB's tag_data entity. */ +/** + * @author koushik + * + */ @Path("/tagdata/") public class GetTagDataStatistics implements ServletContextListener { @@ -37,6 +41,12 @@ public class GetTagDataStatistics implements ServletContextListener { private GetTagDataStatisticsService tagDataService; + /** + * @param crisisCode + * @param granularity + * @param startTime + * @return Tag counts for individual attributes of a crisis, only for the specified granularity and since 'startTime' + */ @GET @Path("/getlabelsum/{crisisCode}/{granularity}") @Produces(MediaType.APPLICATION_JSON) @@ -50,6 +60,12 @@ public Response getTagCountSumForAllAttributesFromTime(@PathParam("crisisCode") } + + /** + * @param crisisCode + * @param startTime + * @return Tag counts over all attributes for a crisis, sorted by granularity levels + */ @GET @Path("/getlabelsum/{crisisCode}") @Produces(MediaType.APPLICATION_JSON) From a5bfee7d9c32ad344369bfe34e3715891c4db0d2 Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Wed, 29 Jul 2015 13:34:58 +0300 Subject: [PATCH 12/44] aidr-analytics config data cleanup; modified single tweet injection in aidr-output --- .../qa/qcri/aidr/analysis/utils/ChannelBufferManager.java | 7 +------ aidr-analytics/src/main/resources/config.properties | 4 ---- .../src/test/java/qa/qcri/aidr/getdata/InjectorConfig.java | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java index c50b78de7..5b48e6a1a 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java @@ -84,10 +84,6 @@ public class ChannelBufferManager { public static String CHANNEL_PREFIX_STRING; //= configProperties.getProperty(OutputConfigurationProperty.TAGGER_CHANNEL_BASENAME)+"."; public static Set subscribedChannels; - // DB access related - //private static DatabaseInterface dbController = null; - private static String managerMainUrl = null; - private static ConcurrentHashMap redisLoadShedder = null; private static ConcurrentHashMap tagDataMap = null; @@ -119,10 +115,9 @@ public void initiateChannelBufferManager(final String channelRegEx) { confDataMap = new ConcurrentHashMap(); channelMap = new ConcurrentHashMap(); - managerMainUrl = configurator.getProperty(OutputConfigurationProperty.MANAGER_URL); logger.info("Initializing channel buffer manager."); System.out.println("[ChannelBufferManager] Initializing channel buffer manager with values: <" + redisHost + ", " + redisPort - + ", " + PERSISTER_LOAD_CHECK_INTERVAL_MINUTES + ", " + PERSISTER_LOAD_LIMIT + ", " + managerMainUrl + ">"); + + ", " + PERSISTER_LOAD_CHECK_INTERVAL_MINUTES + ", " + PERSISTER_LOAD_LIMIT + ">"); bufferSize = -1; executorServicePool = Executors.newCachedThreadPool(); //Executors.newFixedThreadPool(10); // max number of threads diff --git a/aidr-analytics/src/main/resources/config.properties b/aidr-analytics/src/main/resources/config.properties index e8e47e48f..dc902d195 100644 --- a/aidr-analytics/src/main/resources/config.properties +++ b/aidr-analytics/src/main/resources/config.properties @@ -10,8 +10,6 @@ # Redis hostname REDIS_HOST=${REDIS_HOST} -#REST URI of AIDR output -OUTPUT_REST_URI=http://${OUTPUT_API_HOST}:${OUTPUT_API_PORT}/${OUTPUT_API_WEB_APP_CONTEXT}/rest/ ############################# # OPTIONAL configuration @@ -29,7 +27,5 @@ MAX_MESSAGES_COUNT=1000 # INHERITED configuration # Values are inherited from common configuration ############################# -TAGGER_CHANNEL_BASENAME=${TAGGER_CHANNEL_BASENAME} -managerUrl=http://${MANAGER_HOST}:${MANAGER_PORT}/${MANAGER_WEB_APP_CONTEXT} PERSISTER_LOAD_LIMIT=${PERSISTER_LOAD_LIMIT} PERSISTER_LOAD_CHECK_INTERVAL_MINUTES=${PERSISTER_LOAD_CHECK_INTERVAL_MINUTES} diff --git a/aidr-output/src/test/java/qa/qcri/aidr/getdata/InjectorConfig.java b/aidr-output/src/test/java/qa/qcri/aidr/getdata/InjectorConfig.java index 40b7fede3..bf501ad55 100644 --- a/aidr-output/src/test/java/qa/qcri/aidr/getdata/InjectorConfig.java +++ b/aidr-output/src/test/java/qa/qcri/aidr/getdata/InjectorConfig.java @@ -17,7 +17,7 @@ public class InjectorConfig { public final Listcollection_list = Arrays.asList("2014-03-mh370", "2014-04-mers", "2014-05-emsc_landslides_2014", "2014-04-terremoto_chile_2014", "2014-05-serbia_bosnia_floods_may_2014"); public static final String DEFAULT_PERSISTER_FILE_PATH = "/var/www/aidr/data/persister/"; - public final String singleTweet = "{\"filter_level\":\"medium\",\"text\":\"@wongt0n You in Japan now?\",\"contributors\":null,\"geo\":null,\"retweeted\":false,\"in_reply_to_screen_name\":\"wongt0n\",\"truncated\":false,\"entities\":{\"hashtags\":[],\"symbols\":[],\"urls\":[],\"user_mentions\":[{\"id\":123834194,\"indices\":[0,8],\"screen_name\":\"wongt0n\",\"id_str\":\"123834194\",\"name\":\"Lazy Wong~\"}]},\"lang\":\"en\",\"in_reply_to_status_id_str\":\"421190875260014592\",\"id\":421193072316149761,\"aidr\":{\"features\":[{\"words\":[\"\",\"2009\",\"edition\",\"_#mp3\",\"#mp3_#music\",\"408_the\",\"the_end\",\"peas_2009\",\"#mp3\",\"the\",\"edition_black\",\"end_japan\",\"eyed_peas\",\"#music_408\",\"408\",\"japan\",\"japan_edition\",\"black_eyed\",\"#music\",\"eyed\",\"end\",\"black\",\"peas\"],\"type\":\"wordvector\"}],\"crisis_code\":\"japan_chem_explosion\",\"nominal_labels\":[{\"label_name\":\"Not related to crisis\",\"source_id\":289,\"from_human\":false,\"attribute_description\":\"Informative messages enhancing situational awareness, v1.0\",\"label_code\":\"030_info\",\"confidence\":0.54,\"label_description\":\"Not related to the crisis\",\"attribute_code\":\"informative_v2\",\"attribute_name\":\"Informative v1.0\"}, {\"label_name\":\"Not related to crisis\",\"source_id\":289,\"from_human\":false,\"attribute_description\":\"Informative messages enhancing situational awareness, v1.0\",\"label_code\":\"030_info\",\"confidence\":0.8433459674,\"label_description\":\"Not related to the crisis\",\"attribute_code\":\"informative_v1\",\"attribute_name\":\"Informative v1.0\"}, {\"label_name\":\"Not related to crisis\",\"source_id\":289,\"from_human\":false,\"attribute_description\":\"Mock Attribute for Testing, v1.0\",\"label_code\":\"030_not_info\",\"confidence\":0.20,\"label_description\":\"related to the crisis\",\"attribute_code\":\"mock attribute\",\"attribute_name\":\"Mock Attribute\"}],\"doctype\":\"twitter\",\"crisis_name\":\"Explosion at Japan chemical factory\"},\"source\":\"web\",\"in_reply_to_user_id_str\":\"123834194\",\"favorited\":false,\"in_reply_to_status_id\":421190875260014592,\"retweet_count\":0,\"created_at\":\"Thu Jan 09 08:13:48 +0000 2014\",\"in_reply_to_user_id\":123834194,\"favorite_count\":0,\"id_str\":\"421193072316149761\",\"place\":null,\"user\":{\"location\":\"\",\"default_profile\":false,\"profile_background_tile\":true,\"statuses_count\":10045,\"lang\":\"en\",\"profile_link_color\":\"1212E3\",\"profile_banner_url\":\"https://pbs.twimg.com/profile_banners/457120810/1358321901\",\"id\":457120810,\"following\":null,\"protected\":false,\"favourites_count\":7,\"profile_text_color\":\"E60ED4\",\"contributors_enabled\":false,\"verified\":false,\"description\":\"An ordinary SONE who fall in love with Taeyeon+Jessica+Sunny+Tiffany+Hyoyeon+Yuri+Sooyoung+Yoona+Seohyun = SNSD. Spazzing and sharing is my vacation on twitter.\",\"name\":\"~~\uC18C\uC2DC\uB77C\uC11C \uD589\uBCF5\uD574\uC694~~\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_background_color\":\"EDFAFA\",\"created_at\":\"Sat Jan 07 01:51:54 +0000 2012\",\"default_profile_image\":false,\"followers_count\":47,\"geo_enabled\":false,\"profile_image_url_https\":\"https://pbs.twimg.com/profile_images/378800000219857862/9606b10e2dd7d700111f4c5be7384f63_normal.jpeg\",\"profile_background_image_url\":\"http://a0.twimg.com/profile_background_images/889556219/7456374b70ecfea67145b0214f15a988.jpeg\",\"profile_background_image_url_https\":\"https://si0.twimg.com/profile_background_images/889556219/7456374b70ecfea67145b0214f15a988.jpeg\",\"follow_request_sent\":null,\"url\":null,\"utc_offset\":28800,\"time_zone\":\"Kuala Lumpur\",\"notifications\":null,\"profile_use_background_image\":true,\"friends_count\":127,\"profile_sidebar_fill_color\":\"E1D2F5\",\"screen_name\":\"blueagle90\",\"id_str\":\"457120810\",\"profile_image_url\":\"http://pbs.twimg.com/profile_images/378800000219857862/9606b10e2dd7d700111f4c5be7384f63_normal.jpeg\",\"listed_count\":0,\"is_translator\":false},\"coordinates\":null}"; + public final String singleTweet = "{\"filter_level\":\"medium\",\"text\":\"@wongt0n You in Japan now?\",\"contributors\":null,\"geo\":null,\"retweeted\":false,\"in_reply_to_screen_name\":\"wongt0n\",\"truncated\":false,\"entities\":{\"hashtags\":[],\"symbols\":[],\"urls\":[],\"user_mentions\":[{\"id\":123834194,\"indices\":[0,8],\"screen_name\":\"wongt0n\",\"id_str\":\"123834194\",\"name\":\"Lazy Wong~\"}]},\"lang\":\"en\",\"in_reply_to_status_id_str\":\"421190875260014592\",\"id\":421193072316149761,\"aidr\":{\"features\":[{\"words\":[\"\",\"2009\",\"edition\",\"_#mp3\",\"#mp3_#music\",\"408_the\",\"the_end\",\"peas_2009\",\"#mp3\",\"the\",\"edition_black\",\"end_japan\",\"eyed_peas\",\"#music_408\",\"408\",\"japan\",\"japan_edition\",\"black_eyed\",\"#music\",\"eyed\",\"end\",\"black\",\"peas\"],\"type\":\"wordvector\"}],\"crisis_code\":\"japan_chem_explosion\",\"nominal_labels\":[{\"label_name\":\"Not related to crisis\",\"source_id\":289,\"from_human\":false,\"attribute_description\":\"Informative messages enhancing situational awareness, v1.0\",\"label_code\":\"030_info\",\"confidence\":0.54,\"label_description\":\"Not related to the crisis\",\"attribute_code\":\"informative_v2\",\"attribute_name\":\"Informative v1.0\"}, {\"label_name\":\"Not related to crisis\",\"source_id\":289,\"from_human\":false,\"attribute_description\":\"Informative messages enhancing situational awareness, v1.0\",\"label_code\":\"030_info\",\"confidence\":0.8433459674,\"label_description\":\"Not related to the crisis\",\"attribute_code\":\"informative_v1\",\"attribute_name\":\"Informative v1.0\"}, {\"label_name\":\"Not related to crisis\",\"source_id\":289,\"from_human\":false,\"attribute_description\":\"Mock Attribute for Testing, v1.0\",\"label_code\":\"030_info\",\"confidence\":0.40,\"label_description\":\"related to the crisis\",\"attribute_code\":\"mock attribute\",\"attribute_name\":\"Mock Attribute\"}, {\"label_name\":\"Not related to crisis\",\"source_id\":289,\"from_human\":false,\"attribute_description\":\"Mock Attribute for Testing, v1.0\",\"label_code\":\"099_disaster\",\"confidence\":0.79,\"label_description\":\"disaster\",\"attribute_code\":\"mock attribute\",\"attribute_name\":\"Mock Attribute\"}],\"doctype\":\"twitter\",\"crisis_name\":\"Explosion at Japan chemical factory\"},\"source\":\"web\",\"in_reply_to_user_id_str\":\"123834194\",\"favorited\":false,\"in_reply_to_status_id\":421190875260014592,\"retweet_count\":0,\"created_at\":\"Thu Jan 09 08:13:48 +0000 2014\",\"in_reply_to_user_id\":123834194,\"favorite_count\":0,\"id_str\":\"421193072316149761\",\"place\":null,\"user\":{\"location\":\"\",\"default_profile\":false,\"profile_background_tile\":true,\"statuses_count\":10045,\"lang\":\"en\",\"profile_link_color\":\"1212E3\",\"profile_banner_url\":\"https://pbs.twimg.com/profile_banners/457120810/1358321901\",\"id\":457120810,\"following\":null,\"protected\":false,\"favourites_count\":7,\"profile_text_color\":\"E60ED4\",\"contributors_enabled\":false,\"verified\":false,\"description\":\"An ordinary SONE who fall in love with Taeyeon+Jessica+Sunny+Tiffany+Hyoyeon+Yuri+Sooyoung+Yoona+Seohyun = SNSD. Spazzing and sharing is my vacation on twitter.\",\"name\":\"~~\uC18C\uC2DC\uB77C\uC11C \uD589\uBCF5\uD574\uC694~~\",\"profile_sidebar_border_color\":\"FFFFFF\",\"profile_background_color\":\"EDFAFA\",\"created_at\":\"Sat Jan 07 01:51:54 +0000 2012\",\"default_profile_image\":false,\"followers_count\":47,\"geo_enabled\":false,\"profile_image_url_https\":\"https://pbs.twimg.com/profile_images/378800000219857862/9606b10e2dd7d700111f4c5be7384f63_normal.jpeg\",\"profile_background_image_url\":\"http://a0.twimg.com/profile_background_images/889556219/7456374b70ecfea67145b0214f15a988.jpeg\",\"profile_background_image_url_https\":\"https://si0.twimg.com/profile_background_images/889556219/7456374b70ecfea67145b0214f15a988.jpeg\",\"follow_request_sent\":null,\"url\":null,\"utc_offset\":28800,\"time_zone\":\"Kuala Lumpur\",\"notifications\":null,\"profile_use_background_image\":true,\"friends_count\":127,\"profile_sidebar_fill_color\":\"E1D2F5\",\"screen_name\":\"blueagle90\",\"id_str\":\"457120810\",\"profile_image_url\":\"http://pbs.twimg.com/profile_images/378800000219857862/9606b10e2dd7d700111f4c5be7384f63_normal.jpeg\",\"listed_count\":0,\"is_translator\":false},\"coordinates\":null}"; // If wish to use the following single tweet, then comment the above line and uncomment this //public final String singleTweet = "{\"filter_level\":\"medium\",\"retweeted\":false,\"in_reply_to_screen_name\":null,\"possibly_sensitive\":false,\"truncated\":false,\"lang\":\"en\",\"in_reply_to_status_id_str\":null,\"id\":500870968798375936,\"in_reply_to_user_id_str\":null,\"in_reply_to_status_id\":null,\"created_at\":\"Sun Aug 17 05:05:19 +0000 2014\",\"favorite_count\":0,\"place\":null,\"coordinates\":null,\"retweeted_status\":{\"filter_level\":\"low\",\"text\":\"One week until @RunForPalestine Please help them raise $20,000 for urgent need in Palestine!\",\"contributors\":null,\"geo\":null,\"retweeted\":false,\"in_reply_to_screen_name\":null,\"possibly_sensitive\":false,\"truncated\":false,\"entities\":{\"trends\":[],\"hashtags\":[],\"symbols\":[],\"urls\":[],\"user_mentions\":[{\"id\":2643835002,\"indices\":[15,31],\"screen_name\":\"RunForPalestine\",\"id_str\":\"2643835002\",\"name\":\"Run For Palestine \"}]},\"lang\":\"en\",\"in_reply_to_status_id_str\":null,\"id\":500675792943071233,\"source\":\"Twitter Web Client\",\"in_reply_to_user_id_str\":null,\"favorited\":false,\"in_reply_to_status_id\":null,\"retweet_count\":2,\"created_at\":\"Sat Aug 16 16:09:45 +0000 2014\",\"in_reply_to_user_id\":null,\"favorite_count\":0,\"id_str\":\"500675792943071233\",\"place\":null,\"user\":{\"location\":\"Toronto, Ontario, Canada\",\"default_profile\":false,\"profile_background_tile\":false,\"statuses_count\":195,\"lang\":\"en\",\"profile_link_color\":\"1F4BCF\",\"profile_banner_url\":\"https://pbs.twimg.com/profile_banners/2424909001/1408200278\",\"id\":2424909001,\"following\":null,\"protected\":false,\"favourites_count\":61,\"profile_text_color\":\"333333\",\"contributors_enabled\":false,\"description\":\"Hack for Change is a Toronto Hackathon for positive social change in the Middle East - Nov 7 2014 @ Beit Zatoun #Hack4Change\",\"verified\":false,\"name\":\"Hack For Change\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_background_color\":\"C0DEED\",\"created_at\":\"Thu Apr 03 04:39:24 +0000 2014\",\"default_profile_image\":false,\"followers_count\":301,\"geo_enabled\":false,\"profile_image_url_https\":\"https://pbs.twimg.com/profile_images/500654636114526208/_i_PIaJ2_normal.png\",\"profile_background_image_url\":\"http://abs.twimg.com/images/themes/theme1/bg.png\",\"profile_background_image_url_https\":\"https://abs.twimg.com/images/themes/theme1/bg.png\",\"follow_request_sent\":null,\"url\":\"http://www.hack4change.com\",\"utc_offset\":-14400,\"time_zone\":\"Eastern Time (US & Canada)\",\"notifications\":null,\"profile_use_background_image\":true,\"friends_count\":2000,\"profile_sidebar_fill_color\":\"DDEEF6\",\"screen_name\":\"Hack4Change_com\",\"id_str\":\"2424909001\",\"profile_image_url\":\"http://pbs.twimg.com/profile_images/500654636114526208/_i_PIaJ2_normal.png\",\"listed_count\":1,\"is_translator\":false},\"coordinates\":null},\"contributors\":null,\"text\":\"RT @Hack4Change_com: One week until @RunForPalestine Please help them raise $20,000 for urgent need in Palestine!\",\"geo\":null,\"entities\":{\"trends\":[],\"hashtags\":[],\"symbols\":[],\"urls\":[],\"user_mentions\":[{\"id\":2424909001,\"indices\":[3,19],\"screen_name\":\"Hack4Change_com\",\"id_str\":\"2424909001\",\"name\":\"Hack For Change\"},{\"id\":2643835002,\"indices\":[36,52],\"screen_name\":\"RunForPalestine\",\"id_str\":\"2643835002\",\"name\":\"Run For Palestine \"}]},\"aidr\":{\"features\":[{\"words\":[\"need\",\"raise\",\"20000_for\",\"for\",\"one\",\"palestine\",\"help_them\",\"week_until\",\"until\",\"need_in\",\"help\",\"them_raise\",\"please\",\"20000\",\"for_urgent\",\"_one\",\"urgent_need\",\"raise_20000\",\"urgent\",\"until_please\",\"in\",\"them\",\"in_palestine\",\"one_week\",\"week\",\"please_help\"],\"type\":\"wordvector\"}],\"crisis_code\":\"2014-07-palestine_conflict\",\"nominal_labels\":[{\"label_name\":\"Sympathy and Support for Palestine\",\"source_id\":486,\"from_human\":false,\"attribute_description\":\"Opinion divides\",\"label_code\":\"sympathy_for_palestine\",\"confidence\":0.8,\"label_description\":\"If the message shows sympathy or support for Palestine.\",\"attribute_code\":\"opinion_and_support_divide\",\"attribute_name\":\"Opinion and Support Divide\"},{\"label_name\":\"Other relevant information\",\"source_id\":471,\"from_human\":false,\"attribute_description\":\"Themes of palestine conflict\",\"label_code\":\"other\",\"confidence\":0.5,\"label_description\":\"Other relevant information\",\"attribute_code\":\"palestine_conflict\",\"attribute_name\":\"Palestine conflict\"}],\"doctype\":\"twitter\",\"crisis_name\":\"Palestine Conflict\"},\"source\":\"Twitter Web Client\",\"favorited\":false,\"retweet_count\":0,\"in_reply_to_user_id\":null,\"id_str\":\"500870968798375936\",\"user\":{\"location\":\"\",\"default_profile\":true,\"profile_background_tile\":false,\"statuses_count\":3917,\"lang\":\"en\",\"profile_link_color\":\"0084B4\",\"id\":119584881,\"following\":null,\"protected\":false,\"favourites_count\":100,\"profile_text_color\":\"333333\",\"contributors_enabled\":false,\"description\":null,\"verified\":false,\"name\":\"selma\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_background_color\":\"C0DEED\",\"created_at\":\"Thu Mar 04 02:26:54 +0000 2010\",\"default_profile_image\":false,\"followers_count\":63,\"geo_enabled\":false,\"profile_image_url_https\":\"https://pbs.twimg.com/profile_images/495415287248723968/jiP23kQy_normal.jpeg\",\"profile_background_image_url\":\"http://abs.twimg.com/images/themes/theme1/bg.png\",\"profile_background_image_url_https\":\"https://abs.twimg.com/images/themes/theme1/bg.png\",\"follow_request_sent\":null,\"url\":null,\"utc_offset\":null,\"time_zone\":null,\"notifications\":null,\"profile_use_background_image\":true,\"friends_count\":88,\"profile_sidebar_fill_color\":\"DDEEF6\",\"screen_name\":\"sk5t\",\"id_str\":\"119584881\",\"profile_image_url\":\"http://pbs.twimg.com/profile_images/495415287248723968/jiP23kQy_normal.jpeg\",\"listed_count\":0,\"is_translator\":false}}"; From fce88bb01a41db05d238eb343cfd3915bc1e682e Mon Sep 17 00:00:00 2001 From: Koushik Sinha Date: Wed, 29 Jul 2015 13:48:33 +0300 Subject: [PATCH 13/44] changed json to return unix time for startTime & endTime --- .../java/qa/qcri/aidr/analysis/utils/JsonResponse.java | 6 +++--- .../aidr/analysis/utils/OutputConfigurationProperty.java | 8 +++----- aidr-analytics/src/main/resources/config.properties | 1 + 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java index a500f3f14..345e47747 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JsonResponse.java @@ -18,15 +18,15 @@ public JSONObject getNewJsonResponseObject(String crisisCode, String attributeCo json.put("granularity", granularity); if (startTime != null) { - json.put("startTime", new Date(startTime)); + json.put("startTime", startTime); } else { json.put("startTime", null); } if (endTime != null) { - json.put("endTime", new Date(endTime)); + json.put("endTime", endTime); } else { - json.put("endTime", null); + json.put("endTime", startTime+granularity); } return json; } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java index aaa32d2eb..d3dfc90d6 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java @@ -4,11 +4,9 @@ public enum OutputConfigurationProperty implements ConfigurationProperty { - REDIS_HOST("REDIS_HOST"), REDIS_PORT("REDIS_PORT"), LOGGER("logger"), MANAGER_URL( - "managerUrl"), PERSISTER_LOAD_LIMIT("PERSISTER_LOAD_LIMIT"), PERSISTER_LOAD_CHECK_INTERVAL_MINUTES( - "PERSISTER_LOAD_CHECK_INTERVAL_MINUTES"), OUTPUT_REST_URI( - "OUTPUT_REST_URI"), MAX_MESSAGES_COUNT("MAX_MESSAGES_COUNT"), TAGGER_CHANNEL_BASENAME( - "TAGGER_CHANNEL_BASENAME"); + REDIS_HOST("REDIS_HOST"), REDIS_PORT("REDIS_PORT"), LOGGER("logger"), + PERSISTER_LOAD_LIMIT("PERSISTER_LOAD_LIMIT"), PERSISTER_LOAD_CHECK_INTERVAL_MINUTES("PERSISTER_LOAD_CHECK_INTERVAL_MINUTES"), + MAX_MESSAGES_COUNT("MAX_MESSAGES_COUNT"), TAGGER_CHANNEL_BASENAME("TAGGER_CHANNEL_BASENAME"); private final String configurationProperty; diff --git a/aidr-analytics/src/main/resources/config.properties b/aidr-analytics/src/main/resources/config.properties index dc902d195..d51ae4be2 100644 --- a/aidr-analytics/src/main/resources/config.properties +++ b/aidr-analytics/src/main/resources/config.properties @@ -27,5 +27,6 @@ MAX_MESSAGES_COUNT=1000 # INHERITED configuration # Values are inherited from common configuration ############################# +TAGGER_CHANNEL_BASENAME=${TAGGER_CHANNEL_BASENAME} PERSISTER_LOAD_LIMIT=${PERSISTER_LOAD_LIMIT} PERSISTER_LOAD_CHECK_INTERVAL_MINUTES=${PERSISTER_LOAD_CHECK_INTERVAL_MINUTES} From 8216b3380ed60aa4f36287ad95ed89b2918285a3 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Thu, 30 Jul 2015 00:16:14 +0530 Subject: [PATCH 14/44] Sending mails on errors in ajax request done. --- .../manager/controller/TaggerController.java | 78 ++++++++++++++++--- .../aidr/manager/service/TaggerService.java | 1 + .../service/impl/TaggerServiceImpl.java | 26 +++++++ .../controller/AdminHealthController.js | 1 + .../controller/CollectionCreateController.js | 11 ++- .../controller/CollectionDetailsController.js | 20 +++++ .../js/aidrfm/common/AIDRFMFunctions.js | 25 ++++++ .../home/controller/CollectionController.js | 10 ++- .../InteractiveViewDownloadController.js | 2 + .../controller/AttributeDetailsController.js | 1 + .../view/AttributeDetailsMain.js | 3 + .../view/AttributeValuePanel.js | 1 + .../InteractiveViewDownloadController.js | 2 + .../controller/ModelDetailsController.js | 2 + .../NewCustomAttributeController.js | 4 + .../PredictNewAttributeController.js | 2 + .../TaggerCollectionDetailsController.js | 27 ++++++- .../controller/TrainingDataController.js | 4 + .../controller/TrainingExamplesController.js | 5 +- 19 files changed, 203 insertions(+), 22 deletions(-) diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java index 50efb165b..f1a08d5c4 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java @@ -1,30 +1,57 @@ package qa.qcri.aidr.manager.controller; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; + import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.propertyeditors.CustomDateEditor; import org.springframework.stereotype.Controller; import org.springframework.web.bind.WebDataBinder; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.values.DownloadType; import qa.qcri.aidr.dbmanager.dto.taggerapi.TrainingDataDTO; -import qa.qcri.aidr.manager.dto.*; +import qa.qcri.aidr.manager.dto.CrisisRequest; +import qa.qcri.aidr.manager.dto.DateHistory; +import qa.qcri.aidr.manager.dto.ModelHistoryWrapper; +import qa.qcri.aidr.manager.dto.TaggerAttribute; +import qa.qcri.aidr.manager.dto.TaggerCrisis; +import qa.qcri.aidr.manager.dto.TaggerCrisisExist; +import qa.qcri.aidr.manager.dto.TaggerCrisisRequest; +import qa.qcri.aidr.manager.dto.TaggerCrisisType; +import qa.qcri.aidr.manager.dto.TaggerLabel; +import qa.qcri.aidr.manager.dto.TaggerLabelRequest; +import qa.qcri.aidr.manager.dto.TaggerModel; +import qa.qcri.aidr.manager.dto.TaggerModelFamily; +import qa.qcri.aidr.manager.dto.TaggerModelFamilyCollection; +import qa.qcri.aidr.manager.dto.TaggerResponseWrapper; +import qa.qcri.aidr.manager.dto.TaggerUser; +import qa.qcri.aidr.manager.dto.TaggerUserRequest; +import qa.qcri.aidr.manager.dto.TaskAnswer; +import qa.qcri.aidr.manager.dto.TaskAnswerRequest; +import qa.qcri.aidr.manager.dto.TaskInfo; +import qa.qcri.aidr.manager.dto.UpdateCrisisDTO; import qa.qcri.aidr.manager.exception.AidrException; import qa.qcri.aidr.manager.hibernateEntities.AidrCollection; +import qa.qcri.aidr.manager.hibernateEntities.UserEntity; import qa.qcri.aidr.manager.service.CollectionService; import qa.qcri.aidr.manager.service.TaggerService; -import qa.qcri.aidr.manager.util.CollectionStatus; - -import java.text.SimpleDateFormat; -import java.util.*; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; @Controller @@ -962,4 +989,31 @@ public Map updateMicromapperEnabled(@RequestParam Boolean isMicro } } + /** + * @param url + * @param mailType Type of mail (issues/suggestions) + * @return + * @throws Exception + */ + @RequestMapping(value = "/sendEmailService.action", method = RequestMethod.POST) + @ResponseBody + public Map sendEmail(@RequestParam String url, @RequestParam String mailType, @RequestParam String description) throws Exception { + logger.info("In sending email"); + UserEntity userEntity = getAuthenticatedUser(); + userEntity.getUserName(); + String body = "User:-"+userEntity.getUserName() + "\\nURL:-"+url +"\\nRequestHeader:-"+description; + String subject = mailType; + Boolean result = false; + try{ + result = taggerService.sendMailService(subject,body); + if (result) { + return getUIWrapper(null,true); + } else { + return getUIWrapper(false, "Sending Email Failed"); + } + }catch (Exception e) { + e.printStackTrace(); + return getUIWrapper(false, "Sending Email Failed"); + } + } } \ No newline at end of file diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/TaggerService.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/TaggerService.java index 983c63ba5..63d161f2a 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/TaggerService.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/TaggerService.java @@ -134,4 +134,5 @@ public Map generateJsonTweetIdsFilteredLink(String code, public Map downloadHumanLabeledDocumentsByCrisisUserName(String queryString, String crisisCode, String userName, Integer count, String fileType, String contentType) throws AidrException; public Map updateMicromapperEnabled(String code, Boolean isMicromapperEnabled) throws AidrException; + public Boolean sendMailService(String subject, String body); } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/TaggerServiceImpl.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/TaggerServiceImpl.java index 04cf1025a..1c5e98794 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/TaggerServiceImpl.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/TaggerServiceImpl.java @@ -27,6 +27,7 @@ import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Form; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -2166,4 +2167,29 @@ public Map updateMicromapperEnabled(String code, Boolean isMicro e); } } + + @Override + public Boolean sendMailService(String subject, String body){ + Client client = ClientBuilder.newBuilder() + .register(JacksonFeature.class).build(); + Response clientResponse = null; + try { + WebTarget webResource = client.target(taggerMainUrl + + "/misc/sendEmail"); + Form form = new Form(); + form.param("subject", subject); + form.param("body", body); + + clientResponse = webResource.request().post( + Entity.entity(form,MediaType.APPLICATION_FORM_URLENCODED),Response.class); + if (clientResponse.getStatus() != 200) { + logger.warn("Couldn't contact AIDRTaggerAPI for sending error message"); + return false; + } + } catch (Exception e) { + logger.error("Error in contacting AIDRTaggerAPI: " + clientResponse); + return false; + } + return true; + } } diff --git a/aidr-manager/src/main/webapp/resources/js/administration/health/controller/AdminHealthController.js b/aidr-manager/src/main/webapp/resources/js/administration/health/controller/AdminHealthController.js index b04e20259..98c251b31 100644 --- a/aidr-manager/src/main/webapp/resources/js/administration/health/controller/AdminHealthController.js +++ b/aidr-manager/src/main/webapp/resources/js/administration/health/controller/AdminHealthController.js @@ -48,6 +48,7 @@ Ext.define('ADMIN.health.controller.AdminHealthController', { status = 'Running OK'; } else { status = 'Problem: ' + service + ' not running'; + AIDRFMFunctions.reportIssue(response); } if (service === 'tagger') { me.mainComponent.taggerStatus.setText(status, false); diff --git a/aidr-manager/src/main/webapp/resources/js/aidrfm/collection-create/controller/CollectionCreateController.js b/aidr-manager/src/main/webapp/resources/js/aidrfm/collection-create/controller/CollectionCreateController.js index 7a9f981fb..810b3903b 100644 --- a/aidr-manager/src/main/webapp/resources/js/aidrfm/collection-create/controller/CollectionCreateController.js +++ b/aidr-manager/src/main/webapp/resources/js/aidrfm/collection-create/controller/CollectionCreateController.js @@ -221,10 +221,12 @@ Ext.define('AIDRFM.collection-create.controller.CollectionCreateController', { ['Error while starting Collection .', 'Please try again later or contact Support'] ); + AIDRFMFunctions.reportIssue(resp); } }, failure: function () { mask.hide(); + } }); @@ -294,10 +296,10 @@ Ext.define('AIDRFM.collection-create.controller.CollectionCreateController', { headers: { 'Accept': 'application/json' }, - success: function (response) { + success: function (resp) { me.checkCount--; - var response = Ext.decode(response.responseText); + var response = Ext.decode(resp.responseText); if (response.data) { AIDRFMFunctions.setAlert('Error', 'Collection Code already exist. Please select another code'); code.markInvalid("Collection Code already exist. Please select another code"); @@ -324,10 +326,10 @@ Ext.define('AIDRFM.collection-create.controller.CollectionCreateController', { headers: { 'Accept': 'application/json' }, - success: function (response) { + success: function (resp) { me.checkCount--; - var response = Ext.decode(response.responseText); + var response = Ext.decode(resp.responseText); if (response.data) { AIDRFMFunctions.setAlert('Error', [ 'The name of the collection you have selected is already taken. Please enter a more specific name for your collection indicating a more specific time, location, and/or purpose.', @@ -337,6 +339,7 @@ Ext.define('AIDRFM.collection-create.controller.CollectionCreateController', { '"Earthquake in Concepcion, Chile in 2014"', '"Consequences of earthquake in Concepcion, Chile in 2014"' ]); + name.markInvalid("Collection Name already exist. Please select another name"); } else { if (me.checkCount == 0) { diff --git a/aidr-manager/src/main/webapp/resources/js/aidrfm/collection-details/controller/CollectionDetailsController.js b/aidr-manager/src/main/webapp/resources/js/aidrfm/collection-details/controller/CollectionDetailsController.js index 7894a3073..a4c482694 100644 --- a/aidr-manager/src/main/webapp/resources/js/aidrfm/collection-details/controller/CollectionDetailsController.js +++ b/aidr-manager/src/main/webapp/resources/js/aidrfm/collection-details/controller/CollectionDetailsController.js @@ -158,6 +158,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { ['Error while starting Collection .', 'Please try again later or contact Support'] ); + AIDRFMFunctions.reportIssue(resp); } }, failure: function () { @@ -699,6 +700,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { }); } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { @@ -733,6 +735,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { @@ -778,6 +781,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { @@ -811,6 +815,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { me.refreshStatus(id); } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { @@ -888,6 +893,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { } else { AIDRFMFunctions.setAlert("Error", [respObj.message]); + AIDRFMFunctions.reportIssue(response); } editPanelEl.unmask(); @@ -895,6 +901,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { failure: function(response, opts) { AIDRFMFunctions.setAlert("Error", ["An error occurred while updating the collection."]); + AIDRFMFunctions.reportIssue(response); editPanelEl.unmask(); } }); @@ -935,6 +942,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); @@ -969,6 +977,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { document.location.href = BASE_URL + '/protected/' + code + '/tagger-collection-details'; } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); @@ -1014,10 +1023,12 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { } else { me.DetailsComponent.CSVLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate CSV service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.DetailsComponent.CSVLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1030,6 +1041,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); Ext.override(Ext.data.proxy.Server, {timeout: Ext.Ajax.timeout}); Ext.override(Ext.data.Connection, {timeout: Ext.Ajax.timeout}); + } }); }, @@ -1065,10 +1077,12 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { } else { me.DetailsComponent.tweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate Tweet Ids service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.DetailsComponent.tweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1112,10 +1126,12 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { } else { me.DetailsComponent.JSONLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate JSON service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.DetailsComponent.JSONLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1163,10 +1179,12 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { } else { me.DetailsComponent.JsonTweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate JSON Tweet Ids service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.DetailsComponent.JsonTweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1206,6 +1224,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { me.DetailsComponent.managersStore.add(resp.data); } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { @@ -1241,6 +1260,7 @@ Ext.define('AIDRFM.collection-details.controller.CollectionDetailsController', { me.DetailsComponent.managersStore.loadData(resp.data); } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { diff --git a/aidr-manager/src/main/webapp/resources/js/aidrfm/common/AIDRFMFunctions.js b/aidr-manager/src/main/webapp/resources/js/aidrfm/common/AIDRFMFunctions.js index cb6b97c85..0c9138237 100644 --- a/aidr-manager/src/main/webapp/resources/js/aidrfm/common/AIDRFMFunctions.js +++ b/aidr-manager/src/main/webapp/resources/js/aidrfm/common/AIDRFMFunctions.js @@ -39,6 +39,31 @@ Ext.define('AIDRFM.common.AIDRFMFunctions', { msgCt.alignTo(document, 't-t'); Ext.DomHelper.append(msgCt, {html: this.buildMessageBox(status.toLowerCase(), message)}, true).slideIn('t').ghost("t", {delay: delay, remove: true}); }, + + reportIssue : function(resp){ + var mailType="Issue"; + var link=window.location.href; + var description=JSON.stringify(resp.request.options); + Ext.Ajax.request({ + url: '/protected/tagger/sendEmailService.action', + params: { + url: link, + mailType: mailType, + description: description + }, + headers: { + 'Accept': 'application/json' + }, + success: function (response) { + //alert("success"+response.responseText); + // alert(window.location.href); + + }, + failure: function (response) { + // alert("failure"+response.responseText); + } + }); + }, buildMessageBox : function(title, msg) { return [ diff --git a/aidr-manager/src/main/webapp/resources/js/aidrfm/home/controller/CollectionController.js b/aidr-manager/src/main/webapp/resources/js/aidrfm/home/controller/CollectionController.js index 5f1f672f8..bff4a165a 100644 --- a/aidr-manager/src/main/webapp/resources/js/aidrfm/home/controller/CollectionController.js +++ b/aidr-manager/src/main/webapp/resources/js/aidrfm/home/controller/CollectionController.js @@ -104,6 +104,7 @@ Ext.define('AIDRFM.home.controller.CollectionController', { ['Error while starting Collection .', 'Please try again later or contact Support'] ); + AIDRFMFunctions.reportIssue(resp); } } }); @@ -136,6 +137,7 @@ Ext.define('AIDRFM.home.controller.CollectionController', { } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); @@ -178,6 +180,7 @@ Ext.define('AIDRFM.home.controller.CollectionController', { }); } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }}); }, @@ -247,8 +250,8 @@ Ext.define('AIDRFM.home.controller.CollectionController', { headers: { 'Accept': 'application/json' }, - success: function (response) { - var response = Ext.decode(response.responseText); + success: function (resp) { + var response = Ext.decode(resp.responseText); if (response.data && response.data.roles) { var roles = response.data.roles; if (Ext.isArray(roles)) { @@ -260,6 +263,7 @@ Ext.define('AIDRFM.home.controller.CollectionController', { } } else { AIDRFMFunctions.setAlert('Error', 'Collection Code already exist. Please select another code'); + AIDRFMFunctions.reportIssue(resp); } } }); @@ -290,6 +294,7 @@ Ext.define('AIDRFM.home.controller.CollectionController', { me.refreshBothCollections(); } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { @@ -321,6 +326,7 @@ Ext.define('AIDRFM.home.controller.CollectionController', { document.location.href = BASE_URL + '/protected/' + code + '/tagger-collection-details'; } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); diff --git a/aidr-manager/src/main/webapp/resources/js/aidrpublic/interactive-view-download/controller/InteractiveViewDownloadController.js b/aidr-manager/src/main/webapp/resources/js/aidrpublic/interactive-view-download/controller/InteractiveViewDownloadController.js index 2e2d5ade8..f8aa59cd9 100644 --- a/aidr-manager/src/main/webapp/resources/js/aidrpublic/interactive-view-download/controller/InteractiveViewDownloadController.js +++ b/aidr-manager/src/main/webapp/resources/js/aidrpublic/interactive-view-download/controller/InteractiveViewDownloadController.js @@ -389,10 +389,12 @@ Ext.define('AIDRPUBLIC.interactive-view-download.controller.InteractiveViewDownl } else { me.mainComponent.downloadLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate Tweet Ids service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.mainComponent.downloadLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/controller/AttributeDetailsController.js b/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/controller/AttributeDetailsController.js index 8abc88495..ac7d854df 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/controller/AttributeDetailsController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/controller/AttributeDetailsController.js @@ -47,6 +47,7 @@ } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/view/AttributeDetailsMain.js b/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/view/AttributeDetailsMain.js index 587501a19..3f5aebbeb 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/view/AttributeDetailsMain.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/view/AttributeDetailsMain.js @@ -358,6 +358,7 @@ Ext.define('TAGGUI.attribute-details.view.AttributeDetailsMain', { }); } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); me.saveButton.enable(); } }, @@ -397,6 +398,7 @@ Ext.define('TAGGUI.attribute-details.view.AttributeDetailsMain', { } else { AIDRFMFunctions.setAlert("Error", resp.message); me.removeClassifierButton.enable(); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { @@ -471,6 +473,7 @@ Ext.define('TAGGUI.attribute-details.view.AttributeDetailsMain', { me.editButton.show(); } else { AIDRFMFunctions.setAlert("Error", 'Error while updating tag in Classifier.'); + AIDRFMFunctions.reportIssue(resp); } me.cancelButton.enable(); me.deleteButton.enable(); diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/view/AttributeValuePanel.js b/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/view/AttributeValuePanel.js index abcb9ebf5..66f542ed2 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/view/AttributeValuePanel.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/attribute-details/view/AttributeValuePanel.js @@ -171,6 +171,7 @@ Ext.define('TAGGUI.attribute-details.view.AttributeValuePanel', { var response = Ext.decode(resp.responseText); if (!response.success) { AIDRFMFunctions.setAlert("Error", 'Error while updating label in Tagger.'); + AIDRFMFunctions.reportIssue(resp); } } }); diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/interactive-view-download/controller/InteractiveViewDownloadController.js b/aidr-manager/src/main/webapp/resources/js/taggui/interactive-view-download/controller/InteractiveViewDownloadController.js index 2f3c5c710..893d7cd84 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/interactive-view-download/controller/InteractiveViewDownloadController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/interactive-view-download/controller/InteractiveViewDownloadController.js @@ -349,10 +349,12 @@ Ext.define('TAGGUI.interactive-view-download.controller.InteractiveViewDownloadC } else { me.mainComponent.downloadLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate Tweet Ids service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.mainComponent.downloadLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/model-details/controller/ModelDetailsController.js b/aidr-manager/src/main/webapp/resources/js/taggui/model-details/controller/ModelDetailsController.js index c243f64a1..11bfa8792 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/model-details/controller/ModelDetailsController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/model-details/controller/ModelDetailsController.js @@ -126,6 +126,7 @@ Ext.define('TAGGUI.model-details.controller.ModelDetailsController', { } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); @@ -151,6 +152,7 @@ Ext.define('TAGGUI.model-details.controller.ModelDetailsController', { } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/new-custom-attribute/controller/NewCustomAttributeController.js b/aidr-manager/src/main/webapp/resources/js/taggui/new-custom-attribute/controller/NewCustomAttributeController.js index e65f976fb..5a9057cdf 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/new-custom-attribute/controller/NewCustomAttributeController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/new-custom-attribute/controller/NewCustomAttributeController.js @@ -272,6 +272,7 @@ Ext.define('TAGGUI.new-custom-attribute.controller.NewCustomAttributeController' } else { mask.hide(); AIDRFMFunctions.setAlert('Error', 'Error in Tagger while validating attribute code'); + AIDRFMFunctions.reportIssue(response); } } }); @@ -310,6 +311,7 @@ Ext.define('TAGGUI.new-custom-attribute.controller.NewCustomAttributeController' } else { mask.hide(); AIDRFMFunctions.setAlert('Error', 'Error while saving attribute in Tagger'); + AIDRFMFunctions.reportIssue(response); } } }); @@ -340,6 +342,7 @@ Ext.define('TAGGUI.new-custom-attribute.controller.NewCustomAttributeController' } else { mask.hide(); AIDRFMFunctions.setAlert('Error', 'Error while saving labels for attribute in Tagger'); + AIDRFMFunctions.reportIssue(response); } } }); @@ -363,6 +366,7 @@ Ext.define('TAGGUI.new-custom-attribute.controller.NewCustomAttributeController' AIDRFMFunctions.setAlert("Ok", '"' + name + '" has been added to "' + CRISIS_NAME + '" crisis.'); } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/predict-new-attribute/controller/PredictNewAttributeController.js b/aidr-manager/src/main/webapp/resources/js/taggui/predict-new-attribute/controller/PredictNewAttributeController.js index d65929eb5..7feb05915 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/predict-new-attribute/controller/PredictNewAttributeController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/predict-new-attribute/controller/PredictNewAttributeController.js @@ -70,6 +70,7 @@ Ext.define('TAGGUI.predict-new-attribute.controller.PredictNewAttributeControlle } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); @@ -100,6 +101,7 @@ Ext.define('TAGGUI.predict-new-attribute.controller.PredictNewAttributeControlle } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/tagger-collection-details/controller/TaggerCollectionDetailsController.js b/aidr-manager/src/main/webapp/resources/js/taggui/tagger-collection-details/controller/TaggerCollectionDetailsController.js index 26f524ed4..813a8d9ef 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/tagger-collection-details/controller/TaggerCollectionDetailsController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/tagger-collection-details/controller/TaggerCollectionDetailsController.js @@ -258,6 +258,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC CRISIS_TYPE_ID = crisisTypeId; } else { AIDRFMFunctions.setAlert("Error", 'Error while saving crisis.'); + AIDRFMFunctions.reportIssue(resp); } } }); @@ -303,6 +304,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC me.mainComponent.templateSaveButton.setText('Edit',false); } else { AIDRFMFunctions.setAlert("Error", 'Error while updating templateSave.'); + AIDRFMFunctions.reportIssue(resp); } mask.hide(); } @@ -342,6 +344,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { AIDRFMFunctions.setAlert("Error", 'Error while updating templateSave.'); + AIDRFMFunctions.reportIssue(resp); } var run = function (delay) { @@ -392,6 +395,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { mask.hide(); AIDRFMFunctions.setAlert("Error", 'Requst failed : ' + maskText ); + AIDRFMFunctions.reportIssue(resp); } //mask.hide(); // AIDRFMFunctions.setAlert("Info", mask); @@ -407,10 +411,11 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC }, - failure: function () { + failure: function (resp) { mask.hide(); maskText = " Requst failed : " + maskText ; AIDRFMFunctions.setAlert("Error", maskText); + AIDRFMFunctions.reportIssue(resp); } }); @@ -443,6 +448,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { btn.show(); AIDRFMFunctions.setAlert("Error", 'Error while enabling MicroMappers'); + AIDRFMFunctions.reportIssue(resp); } } }); @@ -470,6 +476,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { btn.show(); AIDRFMFunctions.setAlert("Error", 'Error while disabling MicroMappers'); + AIDRFMFunctions.reportIssue(resp); } } }); @@ -537,6 +544,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC AIDRFMFunctions.setAlert("Error", resp.message); removeClassifierButton.disabled = false; removeClassifierButton.classList.remove("disabled"); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { @@ -586,6 +594,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } } else { me.mainComponent.pyBossaLink.setText('
Initializing crowdsourcing task. Please come back in a few minutes.
', false); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { @@ -1112,6 +1121,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { me.mainComponent.CSVLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1152,7 +1162,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC if (resp.data && resp.data != '') { me.mainComponent.tweetsIdsLink.setText('
• Download all tweets (tweet-ids only)
', false); if (resp.message) { - AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.setAlert("Error", resp.message); } } else { me.mainComponent.tweetsIdsLink.setText('
• Download all tweets (tweet-ids only) - Not yet available for this crisis.
', false); @@ -1160,6 +1170,8 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { me.mainComponent.tweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); + } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1204,10 +1216,13 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { me.mainComponent.CSVLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate CSV service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.mainComponent.CSVLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); + } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1255,10 +1270,12 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { me.mainComponent.tweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate Tweet Ids service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.mainComponent.tweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1304,6 +1321,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { me.mainComponent.JSONLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1352,6 +1370,7 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { me.mainComponent.JsonTweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1396,10 +1415,12 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { me.mainComponent.JSONLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate JSON service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.mainComponent.JSONLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); @@ -1447,10 +1468,12 @@ Ext.define('TAGGUI.tagger-collection-details.controller.TaggerCollectionDetailsC } else { me.mainComponent.JsonTweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate Tweet Ids service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.mainComponent.JsonTweetsIdsLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } //Ext.Ajax.timeout = 30000; //Ext.override(Ext.form.Basic, {timeout: Ext.Ajax.timeout/1000}); diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js b/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js index 83171e850..561b30936 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js @@ -96,6 +96,7 @@ Ext.define('TAGGUI.training-data.controller.TrainingDataController', { } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); @@ -179,6 +180,7 @@ Ext.define('TAGGUI.training-data.controller.TrainingDataController', { me.mainComponent.trainingDataStore.load(); } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } } }); @@ -223,10 +225,12 @@ Ext.define('TAGGUI.training-data.controller.TrainingDataController', { } else { me.mainComponent.downloadLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate Tweet Ids service returned empty url. For further inquiries please contact admin."); + AIDRFMFunctions.reportIssue(response); } } else { me.mainComponent.downloadLink.setText('', false); AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } }, failure: function () { diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/training-examples/controller/TrainingExamplesController.js b/aidr-manager/src/main/webapp/resources/js/taggui/training-examples/controller/TrainingExamplesController.js index 3061d7ade..8567eea25 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/training-examples/controller/TrainingExamplesController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/training-examples/controller/TrainingExamplesController.js @@ -64,7 +64,6 @@ Ext.define('TAGGUI.training-examples.controller.TrainingExamplesController', { method: 'GET', params: { id: CRISIS_ID -// id: 117 }, headers: { 'Accept': 'application/json' @@ -115,6 +114,7 @@ Ext.define('TAGGUI.training-examples.controller.TrainingExamplesController', { } } else { AIDRFMFunctions.setAlert("Error", resp.message); + AIDRFMFunctions.reportIssue(response); } AIDRFMFunctions.hideMask(mask); }, @@ -176,7 +176,7 @@ Ext.define('TAGGUI.training-examples.controller.TrainingExamplesController', { me.loadData(); } else { me.loadData(); - //AIDRFMFunctions.setAlert("Info", "Task has been saved."); + AIDRFMFunctions.reportIssue(response); } AIDRFMFunctions.hideMask(mask); }, @@ -223,6 +223,7 @@ Ext.define('TAGGUI.training-examples.controller.TrainingExamplesController', { var resp = Ext.decode(response.responseText); if (!resp.success) { AIDRFMFunctions.setAlert("Error", "Error while skip task."); + AIDRFMFunctions.reportIssue(response); } AIDRFMFunctions.hideMask(mask); }, From 5aaf70c21ec8e093814c24352fb9c1b7f84e8c35 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Thu, 30 Jul 2015 00:21:39 +0530 Subject: [PATCH 15/44] Add a new api to send email --- .../qcri/aidr/predictui/api/MiscResource.java | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java index 6d3c89a33..c69eb428c 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java @@ -6,13 +6,29 @@ import java.text.SimpleDateFormat; import java.util.Calendar; -import java.util.Date; import java.util.List; +import javax.ejb.EJB; +import javax.ejb.Stateless; +import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.FormParam; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; + +import org.apache.log4j.Logger; + import qa.qcri.aidr.common.util.EmailClient; import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentDTO; import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentList; -import qa.qcri.aidr.dbmanager.dto.taggerapi.HumanLabeledDocumentListWrapper; import qa.qcri.aidr.dbmanager.dto.taggerapi.ItemToLabelDTO; import qa.qcri.aidr.dbmanager.dto.taggerapi.TrainingDataDTO; import qa.qcri.aidr.predictui.facade.MiscResourceFacade; @@ -21,16 +37,6 @@ import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; -import javax.ejb.EJB; -import javax.ejb.Stateless; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; - -import org.apache.log4j.Logger; - /** * REST Web Service * @@ -252,6 +258,7 @@ public Response downloadHumanLabeledDocumentsByCrisisIDUserName(String queryStri } } + //Logs error in system events table also @POST @Consumes(MediaType.APPLICATION_FORM_URLENCODED) @Produces(MediaType.TEXT_PLAIN) @@ -259,9 +266,7 @@ public Response downloadHumanLabeledDocumentsByCrisisIDUserName(String queryStri public Response sendErrorEmail(@FormParam("code") String code, @FormParam("module") String module, @FormParam("description") String description) throws Exception { Boolean emailSent = true; try { - Date d = new Date(System.currentTimeMillis()); String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); - String body = time+" "+module +" "+ code +"\n"+ description; EmailClient.sendErrorMail(module, body); } catch (Exception e) { @@ -278,4 +283,20 @@ public Response sendErrorEmail(@FormParam("code") String code, @FormParam("modul } return Response.ok().build(); } + + @POST + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) + @Produces(MediaType.TEXT_PLAIN) + @Path("/sendEmail") + public Response sendEmail(@FormParam("subject") String subject, @FormParam("body") String body) throws Exception { + try { + String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); + EmailClient.sendErrorMail(subject, time + "/n"+body); + } catch (Exception e) { + logger.error("Unable to send email"); + logger.error(e.getMessage()); + return Response.serverError().build(); + } + return Response.ok().build(); + } } From 8d73fe70f7576384a13f2744d4c315572d6019b3 Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Thu, 30 Jul 2015 11:41:51 +0530 Subject: [PATCH 16/44] Commit for #98777744 --- .../output/getdata/ChannelBufferManager.java | 5 ++ .../qcri/aidr/output/stream/AsyncStream.java | 2 + .../utils/OutputConfigurationProperty.java | 2 +- .../aidr/output/utils/OutputErrorHandler.java | 46 +++++++++++++++++++ .../src/main/resources/config.properties | 1 + 5 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 aidr-output/src/main/java/qa/qcri/aidr/output/utils/OutputErrorHandler.java diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java index 379fbc057..987037014 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java @@ -43,6 +43,7 @@ import qa.qcri.aidr.output.utils.JedisConnectionObject; import qa.qcri.aidr.output.utils.OutputConfigurationProperty; import qa.qcri.aidr.output.utils.OutputConfigurator; +import qa.qcri.aidr.output.utils.OutputErrorHandler; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPubSub; import redis.clients.jedis.exceptions.JedisConnectionException; @@ -120,6 +121,7 @@ public void initiateChannelBufferManager(final String channelRegEx) { subscriberJedis = null; isConnected = false; logger.error("Fatal error! Couldn't establish connection to REDIS!", e); + OutputErrorHandler.sendErrorMail(e.getLocalizedMessage(), "Fatal error! Couldn't establish connection to REDIS!"); } if (isConnected) { aidrSubscriber = new RedisSubscriber(); @@ -133,6 +135,7 @@ public void initiateChannelBufferManager(final String channelRegEx) { } catch (Exception e) { isSubscribed = false; logger.error("Fatal exception occurred attempting subscription: " + e.toString()); + OutputErrorHandler.sendErrorMail(e.getLocalizedMessage(), "Fatal error! Couldn't establish connection to REDIS!"); } if (isSubscribed) { subscribedChannels = new ConcurrentHashMap(); @@ -531,6 +534,7 @@ private boolean setupRedisConnection(final String channelRegEx) { subscriberJedis = null; isConnected = false; logger.error("Fatal error! Couldn't establish connection to REDIS!"); + OutputErrorHandler.sendErrorMail(e.getLocalizedMessage(), "Fatal error! Couldn't establish connection to REDIS!"); } if (isConnected) { aidrSubscriber = new RedisSubscriber(); @@ -545,6 +549,7 @@ private boolean setupRedisConnection(final String channelRegEx) { } catch (Exception e) { isSubscribed = false; logger.error("Fatal exception occurred attempting subscription: " + e.toString()); + OutputErrorHandler.sendErrorMail(e.getLocalizedMessage(), "Fatal error! Couldn't establish connection to REDIS!"); } } return false; diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java b/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java index d6e9e68a7..977ca6356 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java @@ -53,6 +53,7 @@ import qa.qcri.aidr.output.utils.JedisConnectionObject; import qa.qcri.aidr.output.utils.OutputConfigurationProperty; import qa.qcri.aidr.output.utils.OutputConfigurator; +import qa.qcri.aidr.output.utils.OutputErrorHandler; import redis.clients.jedis.Jedis; import com.google.gson.JsonObject; @@ -217,6 +218,7 @@ public ChunkedOutput streamChunkedResponse( } catch (Exception e) { // TODO Auto-generated catch block logger.error(channelCode + ": Fatal exception occurred attempting subscription: " + e.toString()); + OutputErrorHandler.sendErrorMail(e.getLocalizedMessage(), channelCode + ": Fatal exception occurred attempting subscription: " + e.toString()); //System.exit(1); } logger.info(channelCode + ": Spawning async response thread"); diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/OutputConfigurationProperty.java b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/OutputConfigurationProperty.java index 890479006..95c57d475 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/OutputConfigurationProperty.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/OutputConfigurationProperty.java @@ -8,7 +8,7 @@ public enum OutputConfigurationProperty implements ConfigurationProperty { "managerUrl"), PERSISTER_LOAD_LIMIT("PERSISTER_LOAD_LIMIT"), PERSISTER_LOAD_CHECK_INTERVAL_MINUTES( "PERSISTER_LOAD_CHECK_INTERVAL_MINUTES"), OUTPUT_REST_URI( "OUTPUT_REST_URI"), MAX_MESSAGES_COUNT("MAX_MESSAGES_COUNT"), TAGGER_CHANNEL_BASENAME( - "TAGGER_CHANNEL_BASENAME"); + "TAGGER_CHANNEL_BASENAME"), TAGGER_REST_URI("TAGGER_REST_URI"); private final String configurationProperty; diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/OutputErrorHandler.java b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/OutputErrorHandler.java new file mode 100644 index 000000000..fe730528e --- /dev/null +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/OutputErrorHandler.java @@ -0,0 +1,46 @@ +/** + * + */ +package qa.qcri.aidr.output.utils; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Form; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.log4j.Logger; +import org.glassfish.jersey.jackson.JacksonFeature; + +/** + * @author Latika + * + */ +public class OutputErrorHandler { + + private static Logger logger = Logger.getLogger(OutputErrorHandler.class); + + public static void sendErrorMail(String code, String errorMsg) { + Response clientResponse = null; + Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build(); + try { + WebTarget webResource = client.target(OutputConfigurator.getInstance().getProperty(OutputConfigurationProperty.TAGGER_REST_URI) + + "/misc/sendErrorEmail"); + + Form form = new Form(); + form.param("module", "AIDROutput"); + form.param("code", code); + form.param("description", errorMsg); + + clientResponse = webResource.request().post( + Entity.entity(form,MediaType.APPLICATION_FORM_URLENCODED),Response.class); + if (clientResponse.getStatus() != 200) { + logger.warn("Couldn't contact AIDRTaggerAPI for sending error message"); + } + } catch (Exception e) { + logger.error("Error in contacting AIDRTaggerAPI: " + clientResponse); + } + } +} diff --git a/aidr-output/src/main/resources/config.properties b/aidr-output/src/main/resources/config.properties index e8e47e48f..e79d2ff83 100644 --- a/aidr-output/src/main/resources/config.properties +++ b/aidr-output/src/main/resources/config.properties @@ -33,3 +33,4 @@ TAGGER_CHANNEL_BASENAME=${TAGGER_CHANNEL_BASENAME} managerUrl=http://${MANAGER_HOST}:${MANAGER_PORT}/${MANAGER_WEB_APP_CONTEXT} PERSISTER_LOAD_LIMIT=${PERSISTER_LOAD_LIMIT} PERSISTER_LOAD_CHECK_INTERVAL_MINUTES=${PERSISTER_LOAD_CHECK_INTERVAL_MINUTES} +TAGGER_REST_URI=http://${TAGGER_API_HOST}:${TAGGER_API_PORT}/${TAGGER_API_WEB_APP_CONTEXT}/rest From a7df70184333bceafafa4d867ef8c5fc762bd23f Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Thu, 30 Jul 2015 14:38:49 +0530 Subject: [PATCH 17/44] Commit for #83021780 : Remove ErrorLog from aidr-tagger-api, aidr-tagger, aidr-db-manager, aidr-task-manager --- .../facade/impl/CoreDBServiceFacadeImp.java | 76 ++++++++----------- .../facade/imp/DocumentResourceFacadeImp.java | 18 +---- .../imp/TaskAssignmentResourceFacadeImp.java | 20 +---- .../api/CrisisManagementResource.java | 17 +---- .../aidr/predictui/api/CrisisResource.java | 47 ++++++------ .../api/CustomUITemplateResource.java | 21 ++--- .../predictui/api/ModelFamilyResource.java | 17 ++--- .../aidr/predictui/api/TaskBufferScanner.java | 9 --- .../predictui/api/TrainingDataResource.java | 8 -- .../qcri/aidr/predictui/api/UserResource.java | 3 - .../CrisisManagementResourceFacadeImp.java | 7 -- .../facade/imp/CrisisResourceImp.java | 23 ++---- .../facade/imp/ModelFamilyFacadeImp.java | 10 +-- .../facade/imp/TaskBufferScannerImp.java | 4 - .../predictui/facade/imp/UserResourceImp.java | 5 -- .../aidr/predict/common/PipelineProcess.java | 6 +- .../common/TaskManagerEntityMapper.java | 26 +------ .../predict/data/DocumentJSONConverter.java | 13 +--- .../aidr/task/ejb/bean/TaskManagerBean.java | 57 +++----------- 19 files changed, 105 insertions(+), 282 deletions(-) diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java index 624543297..97c208e53 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java @@ -9,13 +9,6 @@ package qa.qcri.aidr.dbmanager.ejb.local.facade.impl; -import org.apache.log4j.Logger; -import org.hibernate.*; -import org.hibernate.criterion.*; - -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.dbmanager.ejb.local.facade.CoreDBServiceFacade; - import java.io.Serializable; import java.util.ArrayList; import java.util.List; @@ -24,6 +17,17 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; +import org.apache.log4j.Logger; +import org.hibernate.Criteria; +import org.hibernate.HibernateException; +import org.hibernate.LockOptions; +import org.hibernate.Session; +import org.hibernate.Transaction; +import org.hibernate.criterion.Criterion; +import org.hibernate.criterion.Order; + +import qa.qcri.aidr.dbmanager.ejb.local.facade.CoreDBServiceFacade; + @Stateless(name = "CoreDBServiceFacadeImp") @@ -33,7 +37,6 @@ public class CoreDBServiceFacadeImp entityClass; @@ -64,7 +67,6 @@ public int setEntityManager(EntityManager em) { return 0; } catch (Exception e) { logger.error("EntityManager setting exception : " + em); - logger.error(elog.toStringException(e)); throw new HibernateException("setEntityManager failed"); } } @@ -74,8 +76,7 @@ public Session getCurrentSession() { try { return em.unwrap(Session.class); } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getCurrentSession failed", e); throw new HibernateException("getCurrentSession failed"); } } @@ -86,8 +87,7 @@ public E getById(I id) { try { return (E) getCurrentSession().get(entityClass, id); } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getById failed, id = " + id, e); throw new HibernateException("getById failed, id = " + id); } } @@ -100,8 +100,7 @@ public E getByCriterionID(Criterion criterion) { criteria.add(criterion); return (E) criteria.uniqueResult(); } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getByCriterionID failed, criteria = " + criterion.toString(), e); throw new HibernateException("getByCriterionID failed, criteria = " + criterion.toString()); } } @@ -115,8 +114,7 @@ public E getByCriteria(Criterion criterion) { List fetchedList = criteria.list(); return (fetchedList != null && !fetchedList.isEmpty()) ? (E) fetchedList.get(0) : null; } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getByCriteria failed, criteria = " + criterion.toString(), e); throw new HibernateException("getByCriteria failed, criteria = " + criterion.toString()); } } @@ -131,8 +129,7 @@ public List getAll() { //System.out.println("CoreDBServiceFacade: getAll fetched list size: " + fetchedList.size()); return fetchedList; } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getAll failed", e); throw new HibernateException("getAll failed"); } } @@ -149,8 +146,7 @@ public List getAllByCriteria(Criterion criterion) { fetchedList = criteria.list(); return fetchedList; } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getAllByCriteria failed, criteria = " + criterion.toString(), e); throw new HibernateException("getAllByCriteria failed, criteria = " + criterion.toString()); } } @@ -169,8 +165,7 @@ public List getByCriteriaWithLimit(Criterion criterion, Integer count) { fetchedList = criteria.list(); return fetchedList; } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getByCriteriaWithLimit failed, criteria = " + criterion.toString(), e); throw new HibernateException("getByCriteriaWithLimit failed, criteria = " + criterion.toString()); } } @@ -196,8 +191,7 @@ public List getByCriteriaByOrder(Criterion criterion, String order, String[] fetchedList = criteria.list(); return fetchedList; } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getByCriteriaWithLimit failed, criteria = " + criterion.toString(), e); throw new HibernateException("getByCriteriaByOrder failed, criteria = " + criterion.toString()); } } @@ -228,8 +222,7 @@ public List getByCriteriaWithAliasByOrder(Criterion criterion, String order, fetchedList = criteria.list(); return fetchedList; } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getByCriteriaWithAliasByOrder failed, criteria = " + criterion.toString(), e); throw new HibernateException("getByCriteriaWithAliasByOrder failed, criteria = " + criterion.toString()); } } @@ -260,24 +253,20 @@ public List getByCriteriaWithInnerJoinByOrder(Criterion criterion, String ord fetchedList = criteria.list(); return fetchedList; } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("getByCriteriaWithInnerJoinByOrder failed, criteria = " + criterion.toString(), e); throw new HibernateException("getByCriteriaWithInnerJoinByOrder failed, criteria = " + criterion.toString()); } } @Override public void update(E e) { - Transaction tx = null; try { Session session = getCurrentSession(); session.saveOrUpdate(e); session.flush(); session.evict(e); } catch (Exception ex) { - System.out.println("Unable to update entity: " + e); - logger.error(elog.toStringException(ex)); - ex.printStackTrace(); + logger.error("Update failed", ex); throw new HibernateException("Update failed"); } } @@ -295,7 +284,7 @@ public void update(List entityCollection) { } if (!tx.wasCommitted()) tx.commit(); } catch (Exception ex) { - logger.error(elog.toStringException(ex)); + logger.error("Update list failed", ex); tx.rollback(); throw new HibernateException("Update list failed"); } @@ -312,9 +301,7 @@ public I save(E e) { return id; } catch (Exception ex) { - System.out.println("Unable to save entity: " + e); - logger.error(elog.toStringException(ex)); - ex.printStackTrace(); + logger.error("Save failed", ex); throw new HibernateException("Save failed"); } @@ -329,8 +316,7 @@ public Object merge(E e) { session.evict(e); return o; } catch (Exception ex) { - logger.error(elog.toStringException(ex)); - ex.printStackTrace(); + logger.error("Merge failed", ex); throw new HibernateException("Merge failed"); } @@ -349,7 +335,7 @@ public void merge(List entityCollection) { } if (!tx.wasCommitted()) tx.commit(); } catch (Exception ex) { - logger.error(elog.toStringException(ex)); + logger.error("Merge list failed", ex); tx.rollback(); throw new HibernateException("Merge list failed"); } @@ -369,7 +355,7 @@ public void save(List entityCollection) { } if (!tx.wasCommitted()) tx.commit(); } catch (Exception ex) { - logger.error(elog.toStringException(ex)); + logger.error("Save list failed", ex); tx.rollback(); throw new HibernateException("Save list failed"); } @@ -384,8 +370,7 @@ public void delete(E e) { session.flush(); session.evict(e); } catch (Exception ex) { - logger.error(elog.toStringException(ex)); - ex.printStackTrace(); + logger.error("Delete failed", ex); throw new HibernateException("Delete failed"); } } @@ -404,7 +389,7 @@ public void delete(List entityCollection) { } if (!tx.wasCommitted()) tx.commit(); } catch (Exception ex) { - logger.error(elog.toStringException(ex)); + logger.error("Delete list failed", ex); tx.rollback(); throw new HibernateException("Delete list failed"); } @@ -416,8 +401,7 @@ public void deleteByCriteria(Criterion criterion) { List entityCollection = getAllByCriteria(criterion); delete(entityCollection); } catch (Exception ex) { - logger.error(elog.toStringException(ex)); - ex.printStackTrace(); + logger.error("Delete by criteria failed", ex); throw new HibernateException("Delete by criteria failed"); } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentResourceFacadeImp.java index 1ca3990e4..4a768404f 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentResourceFacadeImp.java @@ -12,34 +12,26 @@ import javax.ejb.Stateless; import org.apache.log4j.Logger; -import org.hibernate.Criteria; import org.hibernate.Hibernate; import org.hibernate.Query; -import org.hibernate.Transaction; import org.hibernate.Session; +import org.hibernate.Transaction; import org.hibernate.criterion.Criterion; import org.hibernate.criterion.Restrictions; -import qa.qcri.aidr.common.exception.AidrException; import qa.qcri.aidr.common.exception.PropertyNotSetException; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.dbmanager.dto.CrisisDTO; import qa.qcri.aidr.dbmanager.dto.DocumentDTO; import qa.qcri.aidr.dbmanager.dto.NominalLabelDTO; import qa.qcri.aidr.dbmanager.ejb.local.facade.impl.CoreDBServiceFacadeImp; import qa.qcri.aidr.dbmanager.ejb.remote.facade.CrisisResourceFacade; import qa.qcri.aidr.dbmanager.ejb.remote.facade.DocumentResourceFacade; import qa.qcri.aidr.dbmanager.ejb.remote.facade.NominalLabelResourceFacade; -import qa.qcri.aidr.dbmanager.entities.misc.Crisis; -import qa.qcri.aidr.dbmanager.entities.model.Model; import qa.qcri.aidr.dbmanager.entities.task.Document; -import qa.qcri.aidr.dbmanager.entities.task.DocumentNominalLabel; @Stateless(name="DocumentResourceFacadeImp") public class DocumentResourceFacadeImp extends CoreDBServiceFacadeImp implements DocumentResourceFacade { private Logger logger = Logger.getLogger("db-manager-log"); - private ErrorLog elog = new ErrorLog(); @EJB CrisisResourceFacade crisisEJB; @@ -99,7 +91,6 @@ public int deleteNoLabelDocument(DocumentDTO document) { logger.info("deleted count = " + deleteCount); } catch (Exception e) { logger.error("deletion query failed, document: " + document.getDocumentID()); - logger.error(elog.toStringException(e)); return 0; } logger.info("deletion success, deleted count = " + deleteCount); @@ -108,7 +99,6 @@ public int deleteNoLabelDocument(DocumentDTO document) { } catch (Exception e) { logger.error("Deletion query failed"); - logger.error(elog.toStringException(e)); return 0; } } @@ -130,7 +120,6 @@ public int deleteNoLabelDocument(List collection) { logger.info("deleted count = " + deleteCount); } catch (Exception e) { logger.error("Collection deletion query failed"); - logger.error(elog.toStringException(e)); } } return deleteCount; @@ -149,7 +138,6 @@ public int deleteUnassignedDocument(DocumentDTO document) { return result; } catch (Exception e) { logger.error("Deletion query failed"); - logger.error(elog.toStringException(e)); return 0; } } @@ -168,7 +156,6 @@ public int deleteUnassignedDocument(Long documentID) { return result; } catch (Exception e) { logger.error("Deletion query failed"); - logger.error(elog.toStringException(e)); return 0; } } @@ -190,7 +177,6 @@ public int deleteUnassignedDocumentCollection(List documentIDList) { } catch (Exception e) { logger.error("[deleteUnassignedDocumentCollection] Collection deletion query failed"); logger.error("Exception", e); - e.printStackTrace(); } } return deleteCount; @@ -246,11 +232,9 @@ public int deleteStaleDocuments(String joinType, String joinTable, String joinCo logger.info("Constructed query: " + deleteQuery.getQueryString()); try { deleteCount = deleteQuery.executeUpdate(); - System.out.println("[deleteStaleDocuments] number of deleted records = " + deleteCount); logger.info("[deleteStaleDocuments] number of deleted records = " + deleteCount); } catch (Exception e) { logger.error("Exception in executing SQL delete stale docs query"); - logger.error(elog.toStringException(e)); } return deleteCount; } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAssignmentResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAssignmentResourceFacadeImp.java index 3964a718b..c44aa07f2 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAssignmentResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAssignmentResourceFacadeImp.java @@ -18,7 +18,6 @@ import org.apache.log4j.Logger; import org.hibernate.criterion.Restrictions; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.DocumentDTO; import qa.qcri.aidr.dbmanager.dto.TaskAssignmentDTO; import qa.qcri.aidr.dbmanager.ejb.local.facade.impl.CoreDBServiceFacadeImp; @@ -29,7 +28,6 @@ public class TaskAssignmentResourceFacadeImp extends CoreDBServiceFacadeImp implements TaskAssignmentResourceFacade { private Logger logger = Logger.getLogger("db-manager-log"); - private ErrorLog elog = new ErrorLog(); public TaskAssignmentResourceFacadeImp() { super(TaskAssignment.class); @@ -53,9 +51,7 @@ public int insertTaskAssignment(List taskList, Long userID) { } return 1; } catch (Exception e) { - logger.error("Error in insert operation!"); - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("Error in insert operation!", e); return 0; } } @@ -73,7 +69,6 @@ public int insertOneTaskAssignment(Long documentID, Long userID) { } } catch (Exception e) { logger.error("Error in insert operation!", e); - e.printStackTrace(); } return 0; } @@ -93,7 +88,6 @@ public int undoTaskAssignment(List taskList, Long userID) { return deleteCount; } catch (Exception e) { logger.error("Error in undo operation!"); - logger.error(elog.toStringException(e)); } } return 0; @@ -117,7 +111,6 @@ public int undoTaskAssignment(Map taskMap) { return deleteCount; } catch (Exception e) { logger.error("Error in undo operation!"); - logger.error(elog.toStringException(e)); } } return 0; @@ -134,7 +127,6 @@ public int undoTaskAssignment(Long documentID, Long userID) { } } catch (Exception e) { logger.error("Error in undo operation!"); - logger.error(elog.toStringException(e)); } return 0; } @@ -156,7 +148,7 @@ public void undoTaskAssignmentByTimer() { } } } catch (Exception e) { - logger.error(elog.toStringException(e)); + logger.error("Error in undoTaskAssignmentByTimer."); } } @@ -170,7 +162,6 @@ private TaskAssignment getTaskAssignment(Long documentID, Long userID) { return t; } catch (Exception e) { logger.error("Error in find operation: documentID = " + documentID + ", userID = " + userID); - logger.error(elog.toStringException(e)); return null; } } @@ -185,7 +176,6 @@ public TaskAssignmentDTO findTaskAssignment(Long documentID, Long userID) { return t != null ? new TaskAssignmentDTO(t) : null; } catch (Exception e) { logger.error("Error in find operation: documentID = " + documentID + ", userID = " + userID); - logger.error(elog.toStringException(e)); return null; } } @@ -202,10 +192,7 @@ public List findTaskAssignmentByID(Long documentID) { } return dtoList; } catch (Exception e) { - logger.error("Error in find operation: documentID = " + documentID); - logger.error(elog.toStringException(e)); - System.out.println("Error in find operation: documentID = " + documentID); - e.printStackTrace(); + logger.error("Error in find operation: documentID = " + documentID, e); return null; } } @@ -218,7 +205,6 @@ public Integer getPendingTaskCount(Long userID) { return (taskAssignments != null ? taskAssignments.size() : 0); } catch (Exception e) { logger.error("Error in find operation: userID = " + userID); - logger.error(elog.toStringException(e)); } return -1; } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisManagementResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisManagementResource.java index ecf2c3a0f..800cfd618 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisManagementResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisManagementResource.java @@ -1,11 +1,7 @@ package qa.qcri.aidr.predictui.api; -import java.util.List; - import javax.ejb.EJB; import javax.ejb.Stateless; -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -15,16 +11,8 @@ import javax.ws.rs.core.UriInfo; import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.predictui.entities.Crisis; -import qa.qcri.aidr.predictui.entities.Document; -import qa.qcri.aidr.predictui.entities.ModelFamily; -import qa.qcri.aidr.predictui.facade.CrisisManagementResourceFacade; -import qa.qcri.aidr.predictui.facade.CrisisResourceFacade; -import qa.qcri.aidr.predictui.facade.DocumentFacade; -import qa.qcri.aidr.predictui.facade.ModelFamilyFacade; -import qa.qcri.aidr.task.ejb.TaskManagerRemote; +import qa.qcri.aidr.predictui.facade.CrisisManagementResourceFacade; @Path("/manage/collection") @Stateless @@ -38,7 +26,6 @@ public class CrisisManagementResource { //private static Logger logger = Logger.getLogger(CrisisManagementResource.class); private static Logger logger = Logger.getLogger(CrisisManagementResource.class); - private static ErrorLog elog = new ErrorLog(); public CrisisManagementResource() { } @@ -52,7 +39,6 @@ public Response trashByCrisisCode(@PathParam("crisisCode") String crisisCode) { return Response.ok(response).build(); } catch (Exception e) { logger.error("Something went wrong in trashing attempt!"); - logger.error(elog.toStringException(e)); return Response.ok("{\"status\": \"FAILED\"}").build(); } } @@ -66,7 +52,6 @@ public Response untrashByCrisisCode(@PathParam("crisisCode") String crisisCode) return Response.ok(response).build(); } catch (Exception e) { logger.error("Something went wrong in trashing attempt!"); - logger.error(elog.toStringException(e)); return Response.ok("{\"status\": \"FAILED\"}").build(); } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisResource.java index 1de3860ba..70b4c9335 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisResource.java @@ -4,21 +4,23 @@ */ package qa.qcri.aidr.predictui.api; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; - -import qa.qcri.aidr.common.exception.PropertyNotSetException; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.dbmanager.dto.CrisisDTO; -import qa.qcri.aidr.predictui.facade.CrisisResourceFacade; -import qa.qcri.aidr.predictui.util.ResponseWrapper; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.ejb.EJB; import javax.ejb.Stateless; -import javax.ws.rs.*; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -26,11 +28,16 @@ import org.apache.log4j.Logger; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import qa.qcri.aidr.common.exception.PropertyNotSetException; +import qa.qcri.aidr.dbmanager.dto.CrisisDTO; +import qa.qcri.aidr.predictui.facade.CrisisResourceFacade; +import qa.qcri.aidr.predictui.util.ResponseWrapper; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; /** * REST Web Service @@ -43,7 +50,6 @@ public class CrisisResource { //private Logger logger = Logger.getLogger(CrisisResource.class.getName()); private Logger logger = Logger.getLogger(CrisisResource.class); - private ErrorLog elog = new ErrorLog(); @Context private UriInfo context; @@ -141,7 +147,6 @@ public Response getCrisesByCodes(List codes) { for (String c: codes) { logger.error("for code: " + c); } - logger.error(elog.toStringException(e)); return Response.ok("Error while getting numbers of classifiers by crisis codes.").build(); } } @@ -199,9 +204,7 @@ public Response addCrisis(CrisisDTO crisis) { System.out.println("Added crisis successfully: id = " + newCrisis.getCrisisID() + ", " + newCrisis.getCode()); return Response.ok(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)).build(); } catch (RuntimeException e) { - e.printStackTrace(); - logger.error("Error while adding Crisis. Possible causes could be duplication of primary key, incomplete data, incompatible data format. For crisis: " + crisis.getCode()); - logger.error(elog.toStringException(e)); + logger.error("Error while adding Crisis. Possible causes could be duplication of primary key, incomplete data, incompatible data format. For crisis: " + crisis.getCode(), e); return Response.ok("Error while adding Crisis. Possible causes could be duplication of primary key, incomplete data, incompatible data format.").build(); } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CustomUITemplateResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CustomUITemplateResource.java index 773cd0024..229aef4ad 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CustomUITemplateResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CustomUITemplateResource.java @@ -1,15 +1,15 @@ package qa.qcri.aidr.predictui.api; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.dbmanager.dto.CustomUiTemplateDTO; -import qa.qcri.aidr.predictui.facade.CustomUITemplateFacade; -import qa.qcri.aidr.predictui.util.ResponseWrapper; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; +import java.util.List; import javax.ejb.EJB; import javax.ejb.Stateless; -import javax.ws.rs.*; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -17,7 +17,11 @@ import org.apache.log4j.Logger; -import java.util.List; +import qa.qcri.aidr.dbmanager.dto.CustomUiTemplateDTO; +import qa.qcri.aidr.predictui.facade.CustomUITemplateFacade; +import qa.qcri.aidr.predictui.util.ResponseWrapper; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; /** * Created with IntelliJ IDEA. @@ -36,7 +40,6 @@ public class CustomUITemplateResource { private CustomUITemplateFacade customUITemplateFacade; private static Logger logger = Logger.getLogger(CustomUITemplateResource.class); - private static ErrorLog elog = new ErrorLog(); public CustomUITemplateResource(){ diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelFamilyResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelFamilyResource.java index f17f204aa..f73154347 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelFamilyResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelFamilyResource.java @@ -7,31 +7,29 @@ import java.util.Collections; import java.util.List; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.dbmanager.dto.ModelFamilyDTO; -import qa.qcri.aidr.predictui.util.ResponseWrapper; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; - import javax.ejb.EJB; import javax.ejb.Stateless; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.UriInfo; -import javax.ws.rs.Path; import javax.ws.rs.GET; import javax.ws.rs.POST; +import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; import org.apache.log4j.Logger; +import qa.qcri.aidr.dbmanager.dto.ModelFamilyDTO; import qa.qcri.aidr.dbmanager.dto.taggerapi.TaggersForCodes; import qa.qcri.aidr.dbmanager.dto.taggerapi.TaggersForCodesRequest; import qa.qcri.aidr.predictui.facade.ModelFamilyFacade; +import qa.qcri.aidr.predictui.util.ResponseWrapper; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; /** * REST Web Service @@ -49,7 +47,6 @@ public class ModelFamilyResource { //private static Logger logger = Logger.getLogger(ModelFamilyResource.class); private static Logger logger = Logger.getLogger("aidr-tagger-api"); - private static ErrorLog elog = new ErrorLog(); public ModelFamilyResource() { } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TaskBufferScanner.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TaskBufferScanner.java index 85514a383..1e26b0ef1 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TaskBufferScanner.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TaskBufferScanner.java @@ -24,8 +24,6 @@ import javax.ejb.Startup; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -36,10 +34,6 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.exception.ConfigurationPropertyFileException; -import qa.qcri.aidr.common.exception.ConfigurationPropertyNotRecognizedException; -import qa.qcri.aidr.common.exception.ConfigurationPropertyNotSetException; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.predictui.facade.TaskBufferScannerFacade; import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; @@ -59,7 +53,6 @@ public TaskBufferScanner() { //private static Logger logger = Logger.getLogger(TaskBufferScanner.class); private static Logger logger = Logger.getLogger(TaskBufferScanner.class); - private static ErrorLog elog = new ErrorLog(); private static ExecutorService executorService = null; private static boolean threadStatus; @@ -120,12 +113,10 @@ public void run() { }); } catch (RejectedExecutionException e) { logger.error("Fatal error executing thread! Terminating."); - logger.error(elog.toStringException(e)); threadStatus = false; return false; } catch (NullPointerException e) { logger.error("Fatal error executing thread! Terminating."); - logger.error(elog.toStringException(e)); threadStatus = false; return false; } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TrainingDataResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TrainingDataResource.java index 63a5ba23c..39b2d8fc5 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TrainingDataResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TrainingDataResource.java @@ -27,7 +27,6 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.CrisisDTO; import qa.qcri.aidr.predictui.facade.CrisisResourceFacade; import qa.qcri.aidr.predictui.util.ResponseWrapper; @@ -53,7 +52,6 @@ public TrainingDataResource() { //private static Logger logger = Logger.getLogger(TrainingDataResource.class); private static Logger logger = Logger.getLogger(TrainingDataResource.class); - private static ErrorLog elog = new ErrorLog(); @GET @Produces(MediaType.APPLICATION_JSON) @@ -65,7 +63,6 @@ public Response getTweetToTag(@PathParam("crisisCode") Long crisisId) { return Response.ok(crisis).build(); } catch (RuntimeException e) { logger.error("Error in getting tweet to tag for crisis: " + crisisId); - logger.error(elog.toStringException(e)); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } @@ -81,7 +78,6 @@ public Response getCrisisByCode(@PathParam("code") String crisisCode) { return Response.ok(crisis).build(); } catch (RuntimeException e) { logger.error("Error in getting crisis by code: " + crisisCode); - logger.error(elog.toStringException(e)); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } @@ -133,7 +129,6 @@ public Response addCrisis(CrisisDTO crisis) { CrisisDTO newCrisis = crisisLocalEJB.addCrisis(crisis); } catch (RuntimeException e) { logger.error("Error while adding crisis: " + crisis.getCode() + ". Possible causes could be duplication of primary key, incomplete data, incompatible data format."); - logger.error(elog.toStringException(e)); return Response.ok("Error while adding Crisis. Possible causes could be duplication of primary key, incomplete data, incompatible data format.").build(); } @@ -150,7 +145,6 @@ public Response editCrisis(CrisisDTO crisis) { return Response.ok(dto).build(); } catch (RuntimeException e) { logger.error("Error in editing crisis: " + crisis.getCode()); - logger.error(elog.toStringException(e)); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } } @@ -188,11 +182,9 @@ public Response getSampleCountThreshold(){ } catch (FileNotFoundException ex1) { logger.error("Couldn't create input stream for file" + TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.AIDR_TAGGER_CONFIG_URL)); - logger.error(elog.toStringException(ex1)); throw new RuntimeException(ex1); } catch (IOException ex2) { logger.error("Couldn't load file" + TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.AIDR_TAGGER_CONFIG_URL)); - logger.error(elog.toStringException(ex2)); throw new RuntimeException(ex2); } catch (NumberFormatException ex3) { logger.error("Error in parsing sampleCountThreshold from: " + prop diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/UserResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/UserResource.java index 2ca43b3ed..262fc1904 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/UserResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/UserResource.java @@ -24,7 +24,6 @@ import org.apache.log4j.Logger; import qa.qcri.aidr.common.exception.PropertyNotSetException; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.UsersDTO; import qa.qcri.aidr.predictui.facade.UserResourceFacade; import qa.qcri.aidr.predictui.util.ResponseWrapper; @@ -49,7 +48,6 @@ public class UserResource { //private Logger logger = Logger.getLogger(UserResource.class.getName()); private Logger logger = Logger.getLogger(UserResource.class); - private ErrorLog elog = new ErrorLog(); public UserResource() { } @@ -111,7 +109,6 @@ public Response findAllUsers() { return Response.ok(mapper.writeValueAsString(users)).build(); } catch (Exception e) { logger.error("Error in getting all users"); - logger.error(elog.toStringException(e)); return Response.ok(new ArrayList()).build(); } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisManagementResourceFacadeImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisManagementResourceFacadeImp.java index 97f5a88f8..2bf2c5afb 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisManagementResourceFacadeImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisManagementResourceFacadeImp.java @@ -8,7 +8,6 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.CrisisDTO; import qa.qcri.aidr.dbmanager.dto.DocumentDTO; import qa.qcri.aidr.dbmanager.dto.ModelFamilyDTO; @@ -35,7 +34,6 @@ public class CrisisManagementResourceFacadeImp implements CrisisManagementResour private TaskManagerRemote taskManager; private static Logger logger = Logger.getLogger(CrisisManagementResource.class); - private static ErrorLog elog = new ErrorLog(); @Override public String trashByCrisisCode(String crisisCode) { @@ -92,7 +90,6 @@ public String trashByCrisisCode(String crisisCode) { taskManager.deleteTask(associatedDocuments); } catch (Exception e) { logger.error("Error in deleting document set"); - logger.error(elog.toStringException(e)); } List temp = remoteDocumentEJB.findUnLabeledDocumentsByCrisisID(crisis.getCrisisID()); logger.info("Post Trashing: found for " + crisisCode + ", unlabeled docs after delete = " + temp.size()); @@ -106,7 +103,6 @@ public String trashByCrisisCode(String crisisCode) { } } catch (Exception e) { logger.error("Something went wrong in trashing attempt!"); - logger.error(elog.toStringException(e)); StringBuilder sb = new StringBuilder().append("{\"status\": \"FAILED\"}"); return sb.toString(); } @@ -131,8 +127,6 @@ public String untrashByCrisisCode(String crisisCode) { } } catch (Exception e) { logger.error("Could not retrieve crisis to untrash: " + crisisCode); - logger.error(elog.toStringException(e)); - return sb.append("{\"status\": \"FAILED\"}").toString(); } try { @@ -156,7 +150,6 @@ public String untrashByCrisisCode(String crisisCode) { return sb.append("{\"status\": \"UNTRASHED\"}").toString(); } catch (Exception e) { logger.error("Something went wrong in untrashing attempt!"); - logger.error(elog.toStringException(e)); return sb.append("{\"status\": \"FAILED\"}").toString(); } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisResourceImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisResourceImp.java index e3c7f2e99..e98d123ff 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisResourceImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisResourceImp.java @@ -13,7 +13,6 @@ import org.apache.log4j.Logger; import qa.qcri.aidr.common.exception.PropertyNotSetException; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.CrisisDTO; import qa.qcri.aidr.predictui.facade.CrisisResourceFacade; @@ -25,7 +24,6 @@ public class CrisisResourceImp implements CrisisResourceFacade { //private static Logger logger = Logger.getLogger(CrisisResourceImp.class); private static Logger logger = Logger.getLogger(CrisisResourceImp.class); - private static ErrorLog elog = new ErrorLog(); //@PersistenceContext(unitName = "qa.qcri.aidr.predictui-EJBS") //private EntityManager em; @@ -37,8 +35,7 @@ public CrisisDTO addCrisis(CrisisDTO crisis) { try { return remoteCrisisEJB.addCrisis(crisis); } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in addCrisis.", e); } return null; } @@ -47,8 +44,7 @@ public CrisisDTO getCrisisByID(Long id) { try { return remoteCrisisEJB.findCrisisByID(id); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getCrisisByID for id : " + id, e); } return null; } @@ -57,8 +53,7 @@ public CrisisDTO getCrisisByCode(String code) { try { return remoteCrisisEJB.getCrisisByCode(code); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getCrisisByCode for code : " + code, e); } return null; } @@ -68,8 +63,7 @@ public CrisisDTO editCrisis(CrisisDTO crisis) { CrisisDTO editedDTO = remoteCrisisEJB.editCrisis(crisis); return editedDTO; } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in editCrisis.", e); } return null; } @@ -78,8 +72,7 @@ public List getAllCrisis() { try { return remoteCrisisEJB.getAllCrisis(); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getAllCrisis.", e); } return null; } @@ -88,8 +81,7 @@ public List getAllCrisisByUserID(Long userID){ try { return remoteCrisisEJB.getAllCrisisByUserID(userID); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getAllCrisisByUserID for userID : " + userID, e); } return null; } @@ -98,8 +90,7 @@ public Boolean isCrisisExists(String crisisCode) { try { return remoteCrisisEJB.isCrisisExists(crisisCode); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in isCrisisExists for code : " + crisisCode, e); } return false; } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/ModelFamilyFacadeImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/ModelFamilyFacadeImp.java index 69414d979..0dcf45fc7 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/ModelFamilyFacadeImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/ModelFamilyFacadeImp.java @@ -4,7 +4,6 @@ */ package qa.qcri.aidr.predictui.facade.imp; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.predictui.facade.*; import java.util.*; @@ -29,7 +28,6 @@ public class ModelFamilyFacadeImp implements ModelFamilyFacade { private static Logger logger = Logger.getLogger(ModelFamilyFacadeImp.class); - private static ErrorLog elog = new ErrorLog(); @EJB private qa.qcri.aidr.dbmanager.ejb.remote.facade.ModelFamilyResourceFacade remoteModelFamilyEJB; @@ -43,7 +41,7 @@ public List getAllModelFamilies() { try { modelFamilyDTOList = remoteModelFamilyEJB.getAllModelFamilies(); } catch (PropertyNotSetException pe) { - pe.printStackTrace(); + logger.error("Error in fetching all model families.", pe); } return modelFamilyDTOList; } @@ -65,7 +63,7 @@ public List getAllModelFamiliesByCrisis(Long crisisID) { try { modelFamilyDTOList = remoteModelFamilyEJB.getAllModelFamiliesByCrisis(crisisID); } catch (PropertyNotSetException pe) { - pe.printStackTrace(); + logger.error("Error in fetching model familes for crisis : " + crisisID, pe); } return modelFamilyDTOList; @@ -76,7 +74,7 @@ public boolean addCrisisAttribute(ModelFamilyDTO modelFamily) { try { return remoteModelFamilyEJB.addCrisisAttribute(modelFamily); } catch (PropertyNotSetException pe) { - pe.printStackTrace(); + logger.error("Error in adding crisis attribute.", pe); } return false; } @@ -86,7 +84,7 @@ public boolean deleteModelFamily(Long modelFamilyID) { try { return remoteModelFamilyEJB.deleteModelFamily(modelFamilyID); } catch (PropertyNotSetException pe) { - pe.printStackTrace(); + logger.error("Error in deleting model family for modelFamilyID : "+ modelFamilyID, pe); } return false; } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/TaskBufferScannerImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/TaskBufferScannerImp.java index 6919f9c3b..b15eee713 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/TaskBufferScannerImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/TaskBufferScannerImp.java @@ -5,7 +5,6 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.predictui.facade.TaskBufferScannerFacade; import qa.qcri.aidr.task.ejb.TaskManagerRemote; @@ -19,7 +18,6 @@ public class TaskBufferScannerImp implements TaskBufferScannerFacade { //private static Logger logger = Logger.getLogger(TaskBufferScannerImp.class); private static Logger logger = Logger.getLogger(TaskBufferScannerImp.class); - private static ErrorLog elog = new ErrorLog(); //@PersistenceContext(unitName = "qa.qcri.aidr.predictui-EJBS") //private EntityManager em; @@ -35,7 +33,6 @@ public void ScanTaskBuffer(final String maxTaskAge, final String scanInterval) { logger.info("number of deleted stale records = " + result); } catch (Exception e) { logger.error("Exception in executing SQL delete stale docs query"); - logger.error(elog.toStringException(e)); } try { int result = taskManager.deleteStaleTasks("JOIN", @@ -44,7 +41,6 @@ public void ScanTaskBuffer(final String maxTaskAge, final String scanInterval) { logger.info("number of deleted no answer records = " + result); } catch (Exception e) { logger.error("Exception in executing SQL delete no answer docs query"); - logger.error(elog.toStringException(e)); } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/UserResourceImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/UserResourceImp.java index fee70ee81..598fa8639 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/UserResourceImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/UserResourceImp.java @@ -11,12 +11,8 @@ import javax.ejb.Stateless; import org.apache.log4j.Logger; -import org.hibernate.HibernateException; -import qa.qcri.aidr.common.exception.PropertyNotSetException; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.UsersDTO; -import qa.qcri.aidr.dbmanager.entities.model.NominalAttribute; import qa.qcri.aidr.predictui.facade.UserResourceFacade; /** @@ -30,7 +26,6 @@ public class UserResourceImp implements UserResourceFacade { //private EntityManager em; private static Logger logger = Logger.getLogger(UserResourceImp.class); - private static ErrorLog elog = new ErrorLog(); @EJB private qa.qcri.aidr.dbmanager.ejb.remote.facade.UsersResourceFacade remoteUsersEJB; diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/PipelineProcess.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/PipelineProcess.java index aab79a1c4..40d9041ae 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/PipelineProcess.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/PipelineProcess.java @@ -7,10 +7,9 @@ import org.apache.log4j.Logger; -import redis.clients.jedis.Jedis; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.predict.DataStore; import qa.qcri.aidr.predict.data.Document; +import redis.clients.jedis.Jedis; /** * Abstract class for simplifying handling of pipeline processes that consume @@ -23,7 +22,6 @@ public abstract class PipelineProcess implements Runnable { private static Logger logger = Logger.getLogger(PipelineProcess.class); - private static ErrorLog elog = new ErrorLog(); static class ExecutionTime { public double dT; @@ -71,7 +69,6 @@ public void run() { item = Serializer.deserialize(byteDoc.get(1)); } catch (ClassNotFoundException | IOException e) { logger.error("Error when deserializing input document."); - logger.error(elog.toStringException(e)); continue; } @@ -89,7 +86,6 @@ public void run() { outputCount++; } catch (IOException e) { logger.error("Error when serializing output document."); - logger.error(elog.toStringException(e)); } } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaskManagerEntityMapper.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaskManagerEntityMapper.java index ccaffcd58..b11a487a1 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaskManagerEntityMapper.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaskManagerEntityMapper.java @@ -1,9 +1,6 @@ package qa.qcri.aidr.predict.common; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; import org.apache.log4j.Logger; @@ -11,21 +8,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.predict.classification.nominal.NominalLabelBC; -import qa.qcri.aidr.predict.dbentities.TaggerDocument; -import qa.qcri.aidr.predict.dbentities.NominalLabel; -import qa.qcri.aidr.predict.dbentities.TaskAssignment; -import qa.qcri.aidr.predict.data.Document; -import qa.qcri.aidr.predict.data.DocumentJSONConverter; -import qa.qcri.aidr.predict.data.Tweet; -import qa.qcri.aidr.predict.featureextraction.FeatureExtractor; -import qa.qcri.aidr.predict.featureextraction.WordSet; - public class TaskManagerEntityMapper { private static Logger logger = Logger.getLogger(TaskManagerEntityMapper.class); - private static ErrorLog elog = new ErrorLog(); public TaskManagerEntityMapper() {} @@ -40,7 +25,6 @@ public E deSerializeList(String jsonString, TypeReference type) { } } catch (IOException e) { logger.error("JSON deserialization exception"); - logger.error(elog.toStringException(e)); System.err.println("JSON deserialization exception"); e.printStackTrace(); @@ -58,10 +42,7 @@ public E deSerialize(String jsonString, Class entityType) { return entity; } } catch (IOException e) { - logger.error("JSON deserialization exception"); - logger.error(elog.toStringException(e)); - System.err.println("JSON deserialization exception"); - e.printStackTrace(); + logger.error("JSON deserialization exception", e); } return null; } @@ -74,10 +55,7 @@ public String serializeTask(E task) { try { if (task != null) jsonString = mapper.writeValueAsString(task); } catch (IOException e) { - logger.error("JSON deserialization exception"); - logger.error(elog.toStringException(e)); - System.err.println("JSON serialization exception"); - e.printStackTrace(); + logger.error("JSON deserialization exception", e); } return jsonString; } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/DocumentJSONConverter.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/DocumentJSONConverter.java index b32cee9c2..199c9ddec 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/DocumentJSONConverter.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/DocumentJSONConverter.java @@ -4,13 +4,12 @@ import java.util.ArrayList; import java.util.HashMap; +import org.apache.commons.lang3.text.translate.UnicodeEscaper; +import org.apache.log4j.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.apache.commons.lang3.text.translate.UnicodeEscaper; -import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.predict.DataStore; import qa.qcri.aidr.predict.classification.DocumentLabel; import qa.qcri.aidr.predict.classification.geo.GeoLabel; @@ -32,7 +31,6 @@ public class DocumentJSONConverter { private static Logger logger = Logger.getLogger(DocumentJSONConverter.class); - private static ErrorLog elog = new ErrorLog(); private static final String name = "DocumentJsonConverter"; private static long lastModelInfoUpdate = 0; @@ -102,7 +100,6 @@ public static SMS parseSMS(JSONObject input) { return sms; } catch (JSONException e) { logger.error("Json exception in parsing tweet: " + input); - logger.error(elog.toStringException(e)); throw new RuntimeException(e); } } @@ -133,7 +130,6 @@ public static Tweet parseTweet(JSONObject input) { return t; } catch (JSONException e) { logger.error("Json exception in parsing tweet: " + input); - logger.error(elog.toStringException(e)); throw new RuntimeException(e); } } @@ -168,7 +164,6 @@ public static String getDocumentSetJson(Document doc) { } catch (RuntimeException e) { logger.error("Exception while converting document to JSON:" + l); - logger.error(elog.toStringException(e)); } } } else { @@ -182,7 +177,6 @@ public static String getDocumentSetJson(Document doc) { return unicodeEscaper.translate(input.toString()); } catch (JSONException e) { logger.error("Error in creating JSON from document: " + doc); - logger.error(elog.toStringException(e)); throw new RuntimeException(e); } } @@ -222,7 +216,6 @@ public static NominalLabelBC parseNominalLabel(JSONObject input) { return l; } catch (JSONException e) { logger.error("Error in parsing nominal label for: " + input); - logger.error(elog.toStringException(e)); throw new RuntimeException(e); } } @@ -242,7 +235,6 @@ public static JSONObject createEmptyLabelJson() { obj.put("from_human", false); } catch (JSONException e) { logger.error("Error in creating empty json object"); - logger.error(elog.toStringException(e)); throw new RuntimeException(e); } return obj; @@ -271,7 +263,6 @@ public static JSONObject getLabelJson(int crisisID, DocumentLabel label) { } } catch (JSONException e) { logger.error("Error in creating json object from: " + label); - logger.error(elog.toStringException(e)); throw new RuntimeException(e); } logger.error("Unsupported label type: " + label.getClass().getSimpleName()); diff --git a/aidr-task-manager/src/main/java/qa/qcri/aidr/task/ejb/bean/TaskManagerBean.java b/aidr-task-manager/src/main/java/qa/qcri/aidr/task/ejb/bean/TaskManagerBean.java index 208c17d0d..0f1b9eac6 100644 --- a/aidr-task-manager/src/main/java/qa/qcri/aidr/task/ejb/bean/TaskManagerBean.java +++ b/aidr-task-manager/src/main/java/qa/qcri/aidr/task/ejb/bean/TaskManagerBean.java @@ -21,12 +21,7 @@ import org.hibernate.criterion.Criterion; import org.hibernate.criterion.Restrictions; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; - import qa.qcri.aidr.common.exception.PropertyNotSetException; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.CrisisDTO; import qa.qcri.aidr.dbmanager.dto.DocumentDTO; import qa.qcri.aidr.dbmanager.dto.DocumentNominalLabelDTO; @@ -35,12 +30,19 @@ import qa.qcri.aidr.dbmanager.dto.TaskAnswerDTO; import qa.qcri.aidr.dbmanager.dto.TaskAssignmentDTO; import qa.qcri.aidr.dbmanager.dto.UsersDTO; -import qa.qcri.aidr.dbmanager.entities.task.*; import qa.qcri.aidr.dbmanager.entities.misc.Crisis; import qa.qcri.aidr.dbmanager.entities.misc.Users; +import qa.qcri.aidr.dbmanager.entities.task.Document; +import qa.qcri.aidr.dbmanager.entities.task.DocumentNominalLabel; +import qa.qcri.aidr.dbmanager.entities.task.TaskAnswer; +import qa.qcri.aidr.dbmanager.entities.task.TaskAssignment; import qa.qcri.aidr.task.common.TrainingDataFetchType; import qa.qcri.aidr.task.ejb.TaskManagerRemote; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + /** * This class implements the TaskManagerRemote interface, providing the business logic for operations * on the document, document_nominal_label, task_answer and task_assignment table - logically grouped as the the 'task related operations'. @@ -98,7 +100,6 @@ public class TaskManagerBean implements TaskManagerRemote private qa.qcri.aidr.dbmanager.ejb.remote.facade.NominalLabelResourceFacade remoteNominalLabelEJB; protected static Logger logger = Logger.getLogger(TaskManagerBean.class); - private ErrorLog elog = new ErrorLog(); private static Object lockObject = new Object(); private static Integer inCS = 0; @@ -139,7 +140,6 @@ public long insertNewTask(T task) { return savedDoc.getDocumentID(); } catch (Exception e) { logger.error("Error in insertion"); - logger.error(elog.toStringException(e)); } return -1; } @@ -161,7 +161,6 @@ public Long saveNewTask(T task, Long crisisID) { return savedDoc.getDocumentID(); } catch (Exception e) { logger.error("Error in insertion"); - logger.error(elog.toStringException(e)); e.printStackTrace(); } return -1L; @@ -179,7 +178,6 @@ public void insertNewTask(List collection) { } } catch (Exception e) { logger.error("Error in collection insertion"); - logger.error(elog.toStringException(e)); } } else { logger.warn("Attempting to insert NULL"); @@ -202,7 +200,6 @@ public List saveNewTasks(List collection, Long crisisID) { } } catch (Exception e) { logger.error("Error in collection insertion"); - logger.error(elog.toStringException(e)); } } else { logger.warn("Attempting to insert NULL"); @@ -220,7 +217,6 @@ public int deleteTaskById(Long id) { return result; } catch(Exception e) { logger.error("Error in deletion"); - logger.error(elog.toStringException(e)); } return 0; } @@ -234,7 +230,6 @@ public int deleteTask(T task) { //documentLocalEJB.delete((Document) task); } catch (Exception e) { logger.error("Error in deletion of task"); - logger.error(elog.toStringException(e)); return 0; } } else { @@ -280,7 +275,6 @@ public int deleteTask(List collection) { return remoteDocumentEJB.deleteNoLabelDocument(dtoList); } catch (Exception e) { logger.error("Error in collection deletion of size: " + collection.size()); - logger.error(elog.toStringException(e)); return 0; } } else { @@ -296,7 +290,6 @@ public int deleteUnassignedTask(T task) { return remoteDocumentEJB.deleteUnassignedDocument((DocumentDTO) task); } catch (Exception e) { logger.error("Error in deletion"); - logger.error(elog.toStringException(e)); return 0; } } else { @@ -318,7 +311,6 @@ public int deleteUnassignedTaskCollection(List collection) { return remoteDocumentEJB.deleteUnassignedDocumentCollection(idList); } catch (Exception e) { logger.error("Error in collection deletion"); - logger.error(elog.toStringException(e)); return 0; } } else { @@ -340,7 +332,6 @@ public int deleteStaleTasks(String joinType, String joinTable, String joinColumn return docDeleteCount; } catch (Exception e) { logger.error("Error in deletion"); - logger.error(elog.toStringException(e)); return 0; } } @@ -366,7 +357,6 @@ public int truncateLabelingTaskBufferForCrisis(final long crisisID, final int ma }*/ } catch (Exception e) { logger.error("Exception in fetching unassigned documents with hasHumaLabels=false"); - logger.error(elog.toStringException(e)); return 0; } @@ -388,7 +378,6 @@ public int truncateLabelingTaskBufferForCrisis(final long crisisID, final int ma return deleteCount; } catch (Exception e) { logger.error("Exception when attempting to batch delete for trimming the document table"); - logger.error(elog.toStringException(e)); } } else { logger.info("No need for truncation: docListSize = " + docList.size() + ", max buffer size = " + maxLength); @@ -408,7 +397,6 @@ public void updateTask(T task) { remoteDocumentEJB.merge(((DocumentDTO) task).toEntity()); } catch (Exception e) { logger.error("failed update"); - logger.error(elog.toStringException(e)); } } @@ -424,7 +412,6 @@ public void updateTaskList(List collection) { remoteDocumentEJB.merge(createDocumentEntityList((List)collection)); } catch (Exception e) { logger.error("failed collection update"); - logger.error(elog.toStringException(e)); } } @@ -437,7 +424,6 @@ public void updateTask(DocumentDTO dto) { remoteDocumentEJB.merge(dto.toEntity()); } catch (Exception e) { logger.error("failed update"); - logger.error(elog.toStringException(e)); } } @@ -451,7 +437,7 @@ public DocumentDTO getNewTask(Long crisisID) { try { return getNewTask(crisisID, null); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error in fetching new task.", e); } return null; } @@ -485,7 +471,6 @@ public DocumentDTO getNewTask(Long crisisID, Criterion criterion) { } } catch (Exception e) { logger.error("Error in getting new Task for crisisID: " + crisisID); - logger.error(elog.toStringException(e)); } return null; } @@ -520,7 +505,6 @@ public List getNewTaskCollection(Long crisisID, Integer count, Stri } } catch (Exception e) { logger.error("Error in getting new Task collection for crisisID: " + crisisID); - logger.error(elog.toStringException(e)); } return null; } @@ -535,7 +519,6 @@ public Boolean isTaskAssigned(E task) { fetchedList= remoteTaskAssignmentEJB.findTaskAssignmentByID(document.getDocumentID()); } catch (Exception e) { logger.error("Error in finding Task"); - logger.error(elog.toStringException(e)); return false; } } @@ -566,7 +549,6 @@ public Boolean isTaskDone(E task) { } } catch (Exception e) { logger.error("Error in finding document"); - logger.error(elog.toStringException(e)); return false; } } @@ -584,7 +566,6 @@ public Boolean isExists(E task) { } } catch (Exception e) { logger.error("Error in finding document"); - logger.error(elog.toStringException(e)); } } return false; @@ -603,7 +584,6 @@ public DocumentDTO getTaskByCriterion(Long crisisID, Criterion criterion) { } } catch (Exception e) { logger.error("Error in finding task"); - logger.error(elog.toStringException(e)); } return null; } @@ -620,7 +600,6 @@ public List getTaskCollectionByCriterion(Long crisisID, Integer cou } } catch (Exception e) { logger.error("Error in finding task"); - logger.error(elog.toStringException(e)); } return null; } @@ -637,7 +616,6 @@ public List getNominalLabelDocumentCollection(Long nominalLabelID) } } catch (Exception e) { logger.error("Error in getting new document collection for nominal Label ID: " + nominalLabelID); - logger.error(elog.toStringException(e)); } return null; } @@ -657,7 +635,6 @@ public DocumentDTO getTaskById(Long id) { return document; } catch (Exception e) { logger.error("Error in finding task"); - logger.error(elog.toStringException(e)); } return null; } @@ -671,7 +648,6 @@ public List getAllTasks() { return createDocumentDTOList(docList); } catch (Exception e) { logger.error("Error in finding task"); - logger.error(elog.toStringException(e)); } return null; } @@ -685,7 +661,6 @@ public String serializeTask(E task) { if (task != null) jsonString = mapper.writeValueAsString(task); } catch (IOException e) { logger.error("JSON serialization exception"); - logger.error(elog.toStringException(e)); } return jsonString; } @@ -704,7 +679,6 @@ public E deSerializeList(String jsonString, TypeReference type) { } } catch (IOException e) { logger.error("JSON deserialization exception"); - logger.error(elog.toStringException(e)); } return null; } @@ -723,7 +697,6 @@ public E deSerialize(String jsonString, Class entityType) { } } catch (IOException e) { logger.error("JSON deserialization exception"); - logger.error(elog.toStringException(e)); } return null; } @@ -844,7 +817,6 @@ public List getAssignedTasksById(Long id) { return docList; } catch (Exception e) { logger.error("Error in serializing collection"); - logger.error(elog.toStringException(e)); } } return null; @@ -858,7 +830,6 @@ public TaskAssignmentDTO getAssignedTaskByUserId(Long id, Long userId) { return assignedUserTask; } catch (Exception e) { logger.error("Error in serializing collection"); - logger.error(elog.toStringException(e)); } } return null; @@ -903,7 +874,6 @@ public Object setTaskParameter(Class entityType, Long id, Map Object setTaskParameter(Class entityType, Long id, Map Object setTaskParameter(Class entityType, Long id, Map Object setTaskParameter(Class entityType, Long id, Map Date: Thu, 30 Jul 2015 14:46:05 +0530 Subject: [PATCH 18/44] Minor changes --- .../java/qa/qcri/aidr/manager/controller/TaggerController.java | 2 +- .../main/webapp/resources/js/aidrfm/common/AIDRFMFunctions.js | 2 +- .../src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java index f1a08d5c4..577d16b1e 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java @@ -1001,7 +1001,7 @@ public Map sendEmail(@RequestParam String url, @RequestParam Str logger.info("In sending email"); UserEntity userEntity = getAuthenticatedUser(); userEntity.getUserName(); - String body = "User:-"+userEntity.getUserName() + "\\nURL:-"+url +"\\nRequestHeader:-"+description; + String body = "User:-"+userEntity.getUserName() + "\nURL:-"+url +"\nRequestHeader:-"+description; String subject = mailType; Boolean result = false; try{ diff --git a/aidr-manager/src/main/webapp/resources/js/aidrfm/common/AIDRFMFunctions.js b/aidr-manager/src/main/webapp/resources/js/aidrfm/common/AIDRFMFunctions.js index 0c9138237..f58e23145 100644 --- a/aidr-manager/src/main/webapp/resources/js/aidrfm/common/AIDRFMFunctions.js +++ b/aidr-manager/src/main/webapp/resources/js/aidrfm/common/AIDRFMFunctions.js @@ -45,7 +45,7 @@ Ext.define('AIDRFM.common.AIDRFMFunctions', { var link=window.location.href; var description=JSON.stringify(resp.request.options); Ext.Ajax.request({ - url: '/protected/tagger/sendEmailService.action', + url: BASE_URL + '/protected/tagger/sendEmailService.action', params: { url: link, mailType: mailType, diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java index c69eb428c..7c5599060 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java @@ -291,7 +291,7 @@ public Response sendErrorEmail(@FormParam("code") String code, @FormParam("modul public Response sendEmail(@FormParam("subject") String subject, @FormParam("body") String body) throws Exception { try { String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); - EmailClient.sendErrorMail(subject, time + "/n"+body); + EmailClient.sendErrorMail(subject, time + "\n"+body); } catch (Exception e) { logger.error("Unable to send email"); logger.error(e.getMessage()); From 9336bf993e26d278d6da27110ee2f1c7875d4ffb Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Thu, 30 Jul 2015 15:01:03 +0530 Subject: [PATCH 19/44] Commit for #83021780 : Remove ErrorLog from aidr-trainer-api --- .../trainer/api/Jedis/JedisDataStore.java | 3 -- .../api/controller/ClientAppController.java | 22 +++++-------- .../controller/ClientAppSourceController.java | 16 ++++----- .../CustomUITemplateController.java | 25 ++++++-------- .../trainer/api/controller/GeoController.java | 26 ++++----------- .../api/controller/TaskAnswerController.java | 18 ++++------ .../controller/TaskAssignmentController.java | 14 ++++---- .../trainer/api/dao/impl/AbstractDaoImpl.java | 33 +++++++++++-------- .../service/impl/TaskAnswerServiceImpl.java | 19 +++++------ .../aidr/trainer/api/util/Communicator.java | 20 ++++------- .../qcri/aidr/trainer/api/util/GISUtil.java | 5 --- .../api/util/TaskManagerEntityMapper.java | 16 +-------- 12 files changed, 81 insertions(+), 136 deletions(-) diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisDataStore.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisDataStore.java index 4520bf279..cc3e71010 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisDataStore.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisDataStore.java @@ -2,7 +2,6 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @@ -19,7 +18,6 @@ public class JedisDataStore { static JedisPool jedisPool; private static Logger logger = Logger.getLogger(JedisDataStore.class); - private static ErrorLog elog = new ErrorLog(); /* REDIS */ public static Jedis getJedisConnection() throws Exception { @@ -32,7 +30,6 @@ public static Jedis getJedisConnection() throws Exception { return jedisPool.getResource(); } catch (Exception e) { logger.error("Could not establish Redis connection. Is the Redis server running?"); - logger.error(elog.toStringException(e)); throw e; } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppController.java index 89f614645..d8328289e 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppController.java @@ -1,25 +1,24 @@ package qa.qcri.aidr.trainer.api.controller; +import java.util.List; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.trainer.api.entity.ClientApp; import qa.qcri.aidr.trainer.api.service.ClientAppService; import qa.qcri.aidr.trainer.api.store.CodeLookUp; import qa.qcri.aidr.trainer.api.store.StatusCodeType; import qa.qcri.aidr.trainer.api.template.ClientAppModel; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import java.util.List; - /** * Created with IntelliJ IDEA. * User: jlucas @@ -34,7 +33,6 @@ public class ClientAppController { private ClientAppService clientAppService; private static Logger logger = Logger.getLogger(ClientAppController.class); - private static ErrorLog elog = new ErrorLog(); @GET @Produces(MediaType.APPLICATION_JSON) @@ -59,7 +57,6 @@ public Response disableClientApps(@PathParam("crisisID") Long crisisID){ } catch(Exception e){ logger.error("disableClientApps exception for crisisID: " + crisisID); - logger.error(elog.toStringException(e)); } @@ -76,7 +73,6 @@ public Response disableClientApp(@PathParam("crisisID") Long crisisID, @PathPara } catch(Exception e){ logger.error("disableClientApps exception in deleting for crisisId:" + crisisID + ", attributeID: " + attributeID); - logger.error(elog.toStringException(e)); } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppSourceController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppSourceController.java index 9a3fd2d47..c72a06ca0 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppSourceController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppSourceController.java @@ -1,5 +1,11 @@ package qa.qcri.aidr.trainer.api.controller; +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + import org.apache.log4j.Logger; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -7,17 +13,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.trainer.api.service.ClientAppSourceService; import qa.qcri.aidr.trainer.api.store.CodeLookUp; import qa.qcri.aidr.trainer.api.store.StatusCodeType; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import java.util.Date; - /** * Created with IntelliJ IDEA. * User: jlucas @@ -30,7 +29,6 @@ public class ClientAppSourceController { protected static Logger logger = Logger.getLogger(ClientAppSourceController.class); - private static ErrorLog elog = new ErrorLog(); @Autowired private ClientAppSourceService clientAppSourceService; @@ -60,8 +58,6 @@ public Response saveAppSource(String data){ catch(Exception e){ returnValue = StatusCodeType.RETURN_FAIL; logger.error("saveAppSource got exception : "); - logger.error(elog.toStringException(e)); - System.out.println("saveAppSource excpetion : " + e ); } return Response.status(CodeLookUp.APP_REQUEST_SUCCESS).entity(returnValue).build(); diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CustomUITemplateController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CustomUITemplateController.java index 3d7d17488..cf6c27d8a 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CustomUITemplateController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CustomUITemplateController.java @@ -1,24 +1,25 @@ package qa.qcri.aidr.trainer.api.controller; +import java.util.List; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + import org.apache.log4j.Logger; -import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.trainer.api.entity.CustomUITemplate; import qa.qcri.aidr.trainer.api.service.CustomUITemplateService; import qa.qcri.aidr.trainer.api.store.CodeLookUp; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; - -import java.util.Date; -import java.util.Iterator; -import java.util.List; - /** * Created with IntelliJ IDEA. * User: jlucas @@ -31,7 +32,6 @@ public class CustomUITemplateController { protected static Logger logger = Logger.getLogger(CustomUITemplateController.class); - private static ErrorLog elog = new ErrorLog(); @Autowired private CustomUITemplateService customUITemplateService; @@ -83,7 +83,6 @@ public void updateWelcomePage(String data){ } catch(Exception e){ logger.debug("updateWelcomePage. Exception: " + data); - logger.error(elog.toStringException(e)); } } @@ -117,8 +116,6 @@ public void updateTutorial(String data){ } catch(Exception e){ logger.debug("updateTutorial. Exception: " + data); - logger.error(elog.toStringException(e)); - } //updateCustomTemplateByAttribute(Long crisisID, Long attributeID, int customUIType, int skinType) @@ -154,8 +151,6 @@ public void updateSkin(String data){ } catch(Exception e){ logger.debug("updateTutorial. Exception: " + data); - logger.error(elog.toStringException(e)); - } //updateCustomTemplateByAttribute(Long crisisID, Long attributeID, int customUIType, int skinType) diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/GeoController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/GeoController.java index 7398e4b85..91a97b265 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/GeoController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/GeoController.java @@ -1,16 +1,7 @@ package qa.qcri.aidr.trainer.api.controller; -import org.apache.log4j.Logger; -import org.json.simple.parser.ParseException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.ResponseBody; - -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.trainer.api.service.GeoService; +import java.text.SimpleDateFormat; +import java.util.Date; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -18,9 +9,11 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; -import java.net.URLDecoder; -import java.text.SimpleDateFormat; -import java.util.Date; +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import qa.qcri.aidr.trainer.api.service.GeoService; /** * Created with IntelliJ IDEA. @@ -37,7 +30,6 @@ public class GeoController { GeoService geoService; private static Logger logger = Logger.getLogger(GeoController.class); - private static ErrorLog elog = new ErrorLog(); @GET @Produces({MediaType.APPLICATION_JSON, MediaType.WILDCARD}) @@ -67,7 +59,6 @@ public String getMapGeoJSONBasedOnDate(@PathParam("lastupdated") String lastupda } catch (Exception e) { logger.error("Exception getMapGeoJSONBasedOnDate"); - logger.error(elog.toStringException(e)); // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. } return returnValue; @@ -85,7 +76,6 @@ public String getMapGeoJSON() { } catch (Exception e) { logger.error("Exception getMapGeoJSON"); - logger.error(elog.toStringException(e)); // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. } return returnValue; @@ -110,7 +100,6 @@ public String getMapGeoJSONPBasedOnDate(@PathParam("lastupdated") String lastupd } catch (Exception e) { logger.error("Exception getMapGeoJSONPBasedOnDate"); - logger.error(elog.toStringException(e)); //e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. } @@ -129,7 +118,6 @@ public String getMapGeoJSONP() { } catch (Exception e) { logger.error("Exception getMapGeoJSONP"); - logger.error(elog.toStringException(e)); // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAnswerController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAnswerController.java index 3f2bb0a28..26ca5b34e 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAnswerController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAnswerController.java @@ -1,17 +1,15 @@ package qa.qcri.aidr.trainer.api.controller; +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.MediaType; + import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.trainer.api.service.TaskAnswerService; -import qa.qcri.aidr.trainer.api.template.TaskAnswerResponse; - -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; - -import java.util.Date; /** * Created with IntelliJ IDEA. @@ -25,7 +23,6 @@ public class TaskAnswerController { protected static Logger logger = Logger.getLogger(TaskAnswerController.class); - private static ErrorLog elog = new ErrorLog(); @Autowired private TaskAnswerService taskAnswerService; @@ -44,12 +41,9 @@ public void saveTaskAnswer(String data){ } catch(Exception e){ logger.error("saveTaskAnswer got exception on: " + data); - logger.error(elog.toStringException(e)); } - - - // below is non transaction block + // below is non transaction block /** try{ TaskAnswerResponse taskAnswerResponse = taskAnswerService.getTaskAnswerResponseData(data); diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAssignmentController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAssignmentController.java index 6c9b6b668..fde2741ce 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAssignmentController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAssignmentController.java @@ -1,18 +1,21 @@ package qa.qcri.aidr.trainer.api.controller; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.trainer.api.service.TaskAssignmentService; import qa.qcri.aidr.trainer.api.store.CodeLookUp; import qa.qcri.aidr.trainer.api.store.StatusCodeType; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - /** * Created with IntelliJ IDEA. * User: jilucas @@ -25,7 +28,6 @@ public class TaskAssignmentController { protected static Logger logger = Logger.getLogger(TaskAssignmentController.class); - private static ErrorLog elog = new ErrorLog(); @Autowired private TaskAssignmentService taskAssignmentService; diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/AbstractDaoImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/AbstractDaoImpl.java index 18dc63269..44c0c7e77 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/AbstractDaoImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/AbstractDaoImpl.java @@ -1,16 +1,22 @@ package qa.qcri.aidr.trainer.api.dao.impl; +import java.io.Serializable; +import java.util.List; + import org.apache.log4j.Logger; -import org.hibernate.*; -import org.hibernate.criterion.*; +import org.hibernate.Criteria; +import org.hibernate.HibernateException; +import org.hibernate.LockOptions; +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.hibernate.criterion.CriteriaSpecification; +import org.hibernate.criterion.Criterion; +import org.hibernate.criterion.Order; +import org.hibernate.criterion.Projections; import org.springframework.beans.factory.annotation.Autowired; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.trainer.api.dao.AbstractDao; -import java.io.Serializable; -import java.util.List; - public abstract class AbstractDaoImpl implements AbstractDao { private Class entityClass; @@ -20,7 +26,6 @@ protected AbstractDaoImpl(Class entityClass) { } private static Logger logger = Logger.getLogger(AbstractDaoImpl.class); - private static ErrorLog elog = new ErrorLog(); @Autowired private SessionFactory sessionFactory; @@ -62,7 +67,7 @@ public List findAll() { //System.out.println("result = " + result); return result; } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in findAll()."); return null; } @@ -83,7 +88,7 @@ public List findByCriteria(Criterion criterion) { try { return criteria.list(); } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in findByCriteria for criteria : " + criteria.toString()); return null; } } @@ -98,7 +103,7 @@ public List getMaxOrderByCriteria(Criterion criterion, String orderBy) { try { return criteria.list(); } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in getMaxOrderByCriteria for criteria : " + criteria.toString()); return null; } } @@ -117,7 +122,7 @@ public List findByCriteriaByOrder(Criterion criterion, String[] orderBy, Inte try { return criteria.list(); } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in findbyCriteriaByOrder for criteria : " + criteria.toString()); return null; } } @@ -138,7 +143,7 @@ public List findByCriteriaWithAliasByOrder(Criterion criterion, String[] orde try { return criteria.list(); } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in findByCriteriaWithAliasByOrder for criteria : " + criteria.toString()); return null; } } @@ -155,7 +160,7 @@ public List findByCriteria(Criterion criterion, Integer count) { try { return criteria.list(); } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in findByCriteria for criteria : " + criteria.toString()); return null; } } @@ -168,7 +173,7 @@ public E findByCriterionID(Criterion criterion) { try { return (E) criteria.uniqueResult(); } catch (HibernateException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in findByCriterionID for criteria : " + criteria.toString()); return null; } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java index 83608d476..8ca16cdec 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java @@ -1,5 +1,7 @@ package qa.qcri.aidr.trainer.api.service.impl; +import java.util.List; + import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -7,19 +9,15 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.dbmanager.dto.DocumentDTO; - -import qa.qcri.aidr.dbmanager.dto.TaskAnswerDTO; -import qa.qcri.aidr.trainer.api.entity.DocumentNominalLabel; import qa.qcri.aidr.task.ejb.TaskManagerRemote; import qa.qcri.aidr.trainer.api.Jedis.JedisNotifier; - +import qa.qcri.aidr.trainer.api.entity.DocumentNominalLabel; import qa.qcri.aidr.trainer.api.entity.TaskAnswer; -import qa.qcri.aidr.trainer.api.service.*; - -import java.util.List; - +import qa.qcri.aidr.trainer.api.service.CrisisService; +import qa.qcri.aidr.trainer.api.service.DocumentNominalLabelService; +import qa.qcri.aidr.trainer.api.service.DocumentService; +import qa.qcri.aidr.trainer.api.service.TaskAnswerService; +import qa.qcri.aidr.trainer.api.service.TaskAssignmentService; import qa.qcri.aidr.trainer.api.template.PybossaTemplate; import qa.qcri.aidr.trainer.api.template.TaskAnswerResponse; @@ -35,7 +33,6 @@ public class TaskAnswerServiceImpl implements TaskAnswerService{ protected static Logger logger = Logger.getLogger(TaskAnswerServiceImpl.class); - private static ErrorLog elog = new ErrorLog(); private JedisNotifier jedisNotifier ; diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/Communicator.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/Communicator.java index 9cd89e75e..af074bf4a 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/Communicator.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/Communicator.java @@ -1,21 +1,19 @@ package qa.qcri.aidr.trainer.api.util; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; + import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.log4j.Logger; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; -import qa.qcri.aidr.common.logging.ErrorLog; - /** * Created with IntelliJ IDEA. * User: jlucas @@ -27,7 +25,6 @@ public class Communicator { // will be placed on config. protected static Logger logger = Logger.getLogger(Communicator.class); - private static ErrorLog elog = new ErrorLog(); public Communicator(){ @@ -67,7 +64,7 @@ public int sendPut(String data, String url) { } }catch (Exception ex) { - logger.error(elog.toStringException(ex)); + logger.warn("Error in processing request for data : " + data + " and url : " + url); } finally { httpClient.getConnectionManager().shutdown(); } @@ -107,7 +104,6 @@ public String deleteGet(String url){ }catch (Exception ex) { logger.error("ex Code deleteGet2: " + url); - logger.error(elog.toStringException(ex)); } finally { httpClient.getConnectionManager().shutdown(); } @@ -157,7 +153,6 @@ public int sendPost(String data, String url) { }catch (Exception ex) { logger.error("ex Code sendPost2: " + data); logger.error("ex Code sendPost3: " + url); - logger.error(elog.toStringException(ex)); } finally { httpClient.getConnectionManager().shutdown(); } @@ -200,7 +195,7 @@ public String sendPostGet(String data, String url) { }catch (Exception ex) { - logger.error(elog.toStringException(ex)); + logger.error("Error in processing request for data : " + data + " and url :" + url); responseOutput.append("Exception Code : " + ex); } finally { @@ -237,7 +232,6 @@ public String sendGet(String url) { }catch (Exception ex) { logger.error("sendGet url = " + url); - logger.error(elog.toStringException(ex)); } return response.toString(); diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/GISUtil.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/GISUtil.java index 5a9e2820c..6167367c7 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/GISUtil.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/GISUtil.java @@ -7,8 +7,6 @@ import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; -import qa.qcri.aidr.common.logging.ErrorLog; - /** * Utility class for handling GIS. * @@ -28,7 +26,6 @@ public class GISUtil { Communicator communicator; private static Logger logger = Logger.getLogger(GISUtil.class); - private static ErrorLog elog = new ErrorLog(); public GISUtil(){ this.parser = new JSONParser(); @@ -86,9 +83,7 @@ public String getDisplayNameWithReverseLookUp(String key) { } catch(Exception e){ logger.error("getDisplayNameWithReverseLookUp exception for key: " + key); - logger.error(elog.toStringException(e)); } - } return info; } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/TaskManagerEntityMapper.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/TaskManagerEntityMapper.java index 065303b68..6e8fa032a 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/TaskManagerEntityMapper.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/TaskManagerEntityMapper.java @@ -1,26 +1,15 @@ package qa.qcri.aidr.trainer.api.util; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; import org.apache.log4j.Logger; +import org.codehaus.jackson.map.DeserializationConfig; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; -import org.codehaus.jackson.map.DeserializationConfig; - -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.trainer.api.entity.Document; -import qa.qcri.aidr.trainer.api.entity.TaskAssignment; -import qa.qcri.aidr.trainer.api.entity.DocumentNominalLabel; -import qa.qcri.aidr.trainer.api.entity.TaskAnswer; -import qa.qcri.aidr.trainer.api.entity.NominalLabel; public class TaskManagerEntityMapper { private static Logger logger = Logger.getLogger(TaskManagerEntityMapper.class); - private static ErrorLog elog = new ErrorLog(); public TaskManagerEntityMapper() {} @@ -35,7 +24,6 @@ public E deSerializeList(String jsonString, TypeReference type) { } } catch (Exception e) { logger.error("JSON deserialization exception"); - logger.error(elog.toStringException(e)); } return null; } @@ -53,7 +41,6 @@ public E deSerialize(String jsonString, Class entityType) { } } catch (Exception e) { logger.error("JSON deserialization exception"); - logger.error(elog.toStringException(e)); } return null; } @@ -67,7 +54,6 @@ public String serializeTask(E task) { if (task != null) jsonString = mapper.writeValueAsString(task); } catch (IOException e) { logger.error("JSON serialization exception"); - logger.error(elog.toStringException(e)); } return jsonString; } From 78d41647fb35827972d11b1ba9931fab5a7d38de Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Thu, 30 Jul 2015 16:05:12 +0530 Subject: [PATCH 20/44] Sending mail if schedule task failed in Manager --- .../java/qa/qcri/aidr/manager/util/ScheduledTask.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/ScheduledTask.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/ScheduledTask.java index a172fda60..bdf8c5cae 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/ScheduledTask.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/ScheduledTask.java @@ -4,8 +4,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; + import qa.qcri.aidr.manager.hibernateEntities.AidrCollection; import qa.qcri.aidr.manager.service.CollectionService; +import qa.qcri.aidr.manager.service.TaggerService; import java.util.Date; import java.util.List; @@ -19,6 +21,9 @@ public class ScheduledTask { @Autowired private CollectionService collectionService; + + @Autowired + private TaggerService taggerService; @Scheduled(fixedDelay = 10 * 60 * 1000) // 10 minutes - in milliseconds private void scheduledTaskUpdateCollections() { @@ -28,6 +33,7 @@ private void scheduledTaskUpdateCollections() { } catch (Exception e) { logger.error("Error while executing update collections scheduled task"); logger.error("exception: ", e); + taggerService.sendMailService("Error in ScheduledTask","Error while executing update collections scheduled task in ScheduledTask.scheduledTaskUpdateCollections"); return; } if (collections != null) { @@ -39,6 +45,7 @@ private void scheduledTaskUpdateCollections() { } catch (Exception e) { logger.error("Error while updating collection with ID: " + item.getId()); logger.error("exception: ", e); + taggerService.sendMailService("Error in ScheduledTask","Error while executing updating collection with ID: " + item.getId() +" in ScheduledTask.scheduledTaskUpdateCollections"); } } } @@ -53,6 +60,7 @@ private void scheduledTaskStopCollections() { } catch (Exception e) { logger.error("Error while executing checking for collections running duration"); logger.error("exception: ", e); + taggerService.sendMailService("Error in ScheduledTask","Error while executing checking for collections running duration in ScheduledTask.scheduledTaskStopCollections"); return; } if (collections != null) { @@ -68,6 +76,7 @@ private void scheduledTaskStopCollections() { } catch (Exception e) { logger.info("Error while stopping collection with ID: " + item.getId()); logger.error("exception: ", e); + taggerService.sendMailService("Error in ScheduledTask","Error while stopping collection with ID: " + item.getId() +" in ScheduledTask.scheduledTaskStopCollections"); } } } From 465e2922f611e533074a31eafaa59c4641600418 Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Thu, 30 Jul 2015 19:33:11 +0530 Subject: [PATCH 21/44] Commit for #83021780 : Remover ErrorLog usage from aidr-common, aidr-persister, aidr-analytics, aidr-collector --- .../aidr/analysis/utils/CommonOperations.java | 8 +-- .../analysis/utils/JedisConnectionObject.java | 8 --- .../api/CollectorManageResource.java | 29 +++++----- .../aidr/collector/api/SMSCollectorAPI.java | 7 +-- .../aidr/common/code/DateFormatConfig.java | 3 -- .../common/filter/FilterQueryMatcher.java | 3 -- .../qa/qcri/aidr/common/logging/ErrorLog.java | 31 ----------- .../facade/imp/CrisisResourceFacadeImp.java | 1 - .../manager/controller/PublicController.java | 48 +++++++++-------- .../manager/controller/TaggerController.java | 14 ++--- .../api/Persister4CollectionAPI.java | 18 +++---- .../persister/api/Persister4CollectorAPI.java | 10 ++-- .../persister/api/Persister4TaggerAPI.java | 5 -- .../collction/CollectionSubscriber.java | 23 ++++---- .../collction/RedisCollectionPersister.java | 4 -- .../collector/CollectorSubscriber.java | 14 ++--- .../collector/RedisCollectorPersister.java | 5 -- .../tagger/RedisTaggerPersister.java | 7 --- .../persister/tagger/TaggerSubscriber.java | 23 +++----- .../qcri/aidr/redis/JedisConnectionPool.java | 3 -- .../qa/qcri/aidr/utils/JsonDeserializer.java | 53 ------------------- .../java/qa/qcri/aidr/utils/ResultStatus.java | 3 -- .../api/NominalAttributeResource.java | 16 +++--- .../aidr/trainer/api/Jedis/JedisNotifier.java | 1 - .../api/service/impl/DocumentServiceImpl.java | 1 - .../service/impl/TaskAnswerServiceImpl.java | 3 ++ .../impl/TaskAssignmentServiceImpl.java | 7 ++- deploy.sh | 46 ++++------------ profiles/dev/config.properties | 39 +++++++------- 29 files changed, 113 insertions(+), 320 deletions(-) delete mode 100644 aidr-common/src/main/java/qa/qcri/aidr/common/logging/ErrorLog.java diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java index 79f6355d6..21d51053c 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/CommonOperations.java @@ -4,22 +4,16 @@ import org.apache.log4j.Logger; import org.hibernate.Session; -import org.hibernate.SessionFactory; - -import qa.qcri.aidr.common.logging.ErrorLog; public class CommonOperations { private Logger logger = Logger.getLogger(CommonOperations.class); - private ErrorLog elog = new ErrorLog(); - public Session getCurrentSession(EntityManager em) { try { return em.unwrap(Session.class); } catch (Exception e) { - logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("Error in getCurrentSession()", e); } return null; } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java index 8b9f3a382..76a0f2249 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java @@ -7,12 +7,8 @@ import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; - - - import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @@ -36,7 +32,6 @@ public class JedisConnectionObject { // Logger setup private static Logger logger = Logger.getLogger(JedisConnectionObject.class); - private static ErrorLog elog = new ErrorLog(); /** * * @param host hostname on which REDIS resides @@ -78,7 +73,6 @@ public JedisConnectionObject(final String host, final int port) { logger.info("New Jedis pool: " + pool); } catch (Exception e) { logger.error("Fatal error! Could not initialize Jedis Pool!"); - logger.error(elog.toStringException(e)); JedisConnectionObject.poolConfig = null; JedisConnectionObject.pool = null; JedisConnectionObject.poolSetup = false; @@ -120,7 +114,6 @@ public Jedis getJedisResource() { subscriberJedis = null; connectionSetup = false; logger.error("Fatal error! Could not get a resource from the pool."); - logger.error(elog.toStringException(e)); } if (subscriberJedis != null) { allotedJedis.put(subscriberJedis, false); // initially nothing assigned @@ -171,7 +164,6 @@ public void returnJedis(Jedis jedisInstance) { } } catch (JedisConnectionException e) { logger.error("JedisConnectionException occurred..."); - logger.error(elog.toStringException(e)); if (jedisInstance != null && jedisInstance.isConnected()) jedisInstance.close(); jedisInstance = null; //JedisConnectionObject.pool.returnBrokenResource(jedisInstance); diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/CollectorManageResource.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/CollectorManageResource.java index 1195d7a06..1c2018435 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/CollectorManageResource.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/CollectorManageResource.java @@ -1,35 +1,33 @@ package qa.qcri.aidr.collector.api; -import com.google.gson.Gson; - -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.WebTarget; - import java.io.BufferedReader; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.util.List; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.UriInfo; -import javax.ws.rs.Path; import javax.ws.rs.GET; +import javax.ws.rs.Path; import javax.ws.rs.Produces; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; import org.apache.log4j.Logger; import org.glassfish.jersey.jackson.JacksonFeature; import qa.qcri.aidr.collector.beans.CollectionTask; import qa.qcri.aidr.collector.beans.CollectorStatus; -import qa.qcri.aidr.collector.utils.CollectorConfigurator; import qa.qcri.aidr.collector.utils.CollectorConfigurationProperty; +import qa.qcri.aidr.collector.utils.CollectorConfigurator; import qa.qcri.aidr.collector.utils.GenericCache; -import qa.qcri.aidr.common.logging.ErrorLog; + +import com.google.gson.Gson; /** * @author Imran @@ -40,7 +38,6 @@ public class CollectorManageResource { private static Logger logger = Logger.getLogger(CollectorManageResource.class.getName()); - private static ErrorLog elog = new ErrorLog(); private static CollectorConfigurator configProperties = CollectorConfigurator.getInstance(); @Context @@ -77,7 +74,6 @@ public String persistRunningCollections() { } catch (IOException e) { logger.error("Error in persisting running collections"); - logger.error(elog.toStringException(e)); } return response; @@ -106,14 +102,14 @@ public String runPersistedCollections() throws InterruptedException { logger.info("Done reading."); } catch (IOException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in reading file."); } finally { try { if (br != null) { br.close(); } } catch (IOException ex) { - logger.error(elog.toStringException(ex)); + logger.error("Error in reading file."); } } @@ -137,7 +133,6 @@ private String runCollection(CollectionTask collection) { return jsonResponse; } catch (Exception e) { logger.error("Could not start collection"); - logger.error(elog.toStringException(e)); return "Could not start collection"; } } diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/SMSCollectorAPI.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/SMSCollectorAPI.java index 0e018b79e..875ea65f3 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/SMSCollectorAPI.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/SMSCollectorAPI.java @@ -26,10 +26,9 @@ import qa.qcri.aidr.collector.beans.ResponseWrapper; import qa.qcri.aidr.collector.beans.SMS; import qa.qcri.aidr.collector.collectors.JedisPublisher; -import qa.qcri.aidr.collector.utils.CollectorConfigurator; import qa.qcri.aidr.collector.utils.CollectorConfigurationProperty; +import qa.qcri.aidr.collector.utils.CollectorConfigurator; import qa.qcri.aidr.collector.utils.GenericCache; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.redis.LoadShedder; /** @@ -41,7 +40,6 @@ public class SMSCollectorAPI { private static Logger logger = Logger.getLogger(SMSCollectorAPI.class.getName()); - private static ErrorLog elog = new ErrorLog(); private static CollectorConfigurator configProperties= CollectorConfigurator.getInstance(); @@ -104,7 +102,6 @@ public Response receive(@PathParam("collection_code") String code, SMS sms) { } } catch (Exception e) { logger.error("Exception in receiving from SMS collection: " + code + ", data: " + sms); - logger.error(elog.toStringException(e)); } } return Response.ok().build(); @@ -138,7 +135,6 @@ public void startPersister(String collectionCode) { logger.info(collectionCode + ": Collector persister response = " + jsonResponse); } catch (RuntimeException e) { logger.error(collectionCode + ": Could not start persister. Is persister running?"); - logger.error(elog.toStringException(e)); } catch (UnsupportedEncodingException e) { // TODO Auto-generated catch block logger.error(collectionCode + ": Unsupported Encoding scheme used"); @@ -155,7 +151,6 @@ public void stopPersister(String collectionCode) { logger.info(collectionCode + ": Collector persister response = " + jsonResponse); } catch (RuntimeException e) { logger.error(collectionCode + ": Could not stop persister. Is persister running?"); - logger.error(elog.toStringException(e)); } catch (UnsupportedEncodingException e) { // TODO Auto-generated catch block logger.error(collectionCode + ": Unsupported Encoding scheme used"); diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/code/DateFormatConfig.java b/aidr-common/src/main/java/qa/qcri/aidr/common/code/DateFormatConfig.java index d796dc695..c877af5bb 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/code/DateFormatConfig.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/code/DateFormatConfig.java @@ -10,8 +10,6 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; - /** * Several operations related to parsing and formatting dates. * @@ -20,7 +18,6 @@ */ public class DateFormatConfig { private static Logger logger = Logger.getLogger(DateFormatConfig.class); - private static ErrorLog elog = new ErrorLog(); public static final String ISODateFormat = "yyyy-MM-dd'T'HH:mm'Z'"; public static final String StandardDateFormat = "EEE MMM dd HH:mm:ss ZZZ yyyy"; diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/FilterQueryMatcher.java b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/FilterQueryMatcher.java index 7566bd727..9572fb6c1 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/FilterQueryMatcher.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/FilterQueryMatcher.java @@ -16,9 +16,6 @@ import com.google.gson.Gson; import qa.qcri.aidr.common.code.DateFormatConfig; -import qa.qcri.aidr.common.logging.ErrorLog; - - public class FilterQueryMatcher { diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/logging/ErrorLog.java b/aidr-common/src/main/java/qa/qcri/aidr/common/logging/ErrorLog.java deleted file mode 100644 index 5ba0d5c06..000000000 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/logging/ErrorLog.java +++ /dev/null @@ -1,31 +0,0 @@ -package qa.qcri.aidr.common.logging; - -import java.io.PrintWriter; -import java.io.StringWriter; - -/** - * A deprecated utility class for error logs - * - * TODO: REMOVE. Remove this class and all references to this class. - * - */ -@Deprecated -public class ErrorLog { - public ErrorLog() {} - - /** - * A deprecated method to create a String from an Exception. - * - * @param e the exception - * @return a String representing the exception - */ - @Deprecated - public String toStringException(Exception e) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - e.printStackTrace(pw); - - StringBuilder retVal = new StringBuilder(new String("Exception:")).append(System.getProperty("line.separator")).append(sw); - return retVal.toString(); - } -} diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java index dcf8d49f3..98155781b 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java @@ -21,7 +21,6 @@ import org.hibernate.criterion.Restrictions; import qa.qcri.aidr.common.exception.PropertyNotSetException; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.CrisisDTO; import qa.qcri.aidr.dbmanager.ejb.local.facade.impl.CoreDBServiceFacadeImp; import qa.qcri.aidr.dbmanager.ejb.remote.facade.CrisisResourceFacade; diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/PublicController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/PublicController.java index 0f6b5eb33..2a0463116 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/PublicController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/PublicController.java @@ -1,5 +1,18 @@ package qa.qcri.aidr.manager.controller; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; + import org.apache.log4j.Logger; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -7,10 +20,13 @@ import org.springframework.beans.propertyeditors.CustomDateEditor; import org.springframework.stereotype.Controller; import org.springframework.web.bind.WebDataBinder; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.common.values.DownloadType; import qa.qcri.aidr.manager.dto.AidrCollectionTotalDTO; import qa.qcri.aidr.manager.dto.TaggerCrisisType; import qa.qcri.aidr.manager.exception.AidrException; @@ -23,20 +39,10 @@ import qa.qcri.aidr.manager.util.CollectionStatus; import qa.qcri.aidr.manager.util.JsonDataValidator; -import javax.ws.rs.Consumes; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; - -import java.text.SimpleDateFormat; -import java.util.*; - @Controller @RequestMapping("public/collection") public class PublicController extends BaseController{ private static Logger logger = Logger.getLogger(PublicController.class); - private static ErrorLog elog = new ErrorLog(); @Autowired private CollectionService collectionService; @@ -158,7 +164,6 @@ public Map findByRequestCode(@QueryParam("code") String code) th return getUIWrapper(data, true); } catch (Exception e) { - logger.error(elog.toStringException(e)); return getUIWrapper(false); } @@ -178,7 +183,7 @@ public Map findAll(@RequestParam Integer start, @RequestParam In return getUIWrapper(data, true); } catch (Exception e) { - logger.error(elog.toStringException(e)); + logger.error("Error in findAll."); return getUIWrapper(false); } @@ -213,7 +218,7 @@ public Map findAllRunning(@RequestParam Integer start, @RequestP return getUIWrapper(dtoList, count.longValue()); } catch (Exception e) { - logger.error(elog.toStringException(e)); + logger.error("Error in findAllRunning."); return getUIWrapper(false); } @@ -243,7 +248,7 @@ public Map findAllRunningWithNoOutput(@RequestParam Integer star return getUIWrapper(dtoList, count.longValue()); } catch (Exception e) { - logger.error(elog.toStringException(e)); + logger.error("Error in findAllRunningWithNoOutput."); return getUIWrapper(false); } @@ -278,7 +283,7 @@ public Map findAllStop(@RequestParam Integer start, @RequestPara return getUIWrapper(dtoList, count.longValue()); } catch (Exception e) { - logger.error(elog.toStringException(e)); + logger.error("Error in "); return getUIWrapper(false); } } @@ -312,7 +317,7 @@ public Map generateTweetIdsLink(@RequestParam String code) throws return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } catch (Exception e) { - logger.error(elog.toStringException(e)); + logger.error("Error in generateTweetIdsLink for codd : " + code); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -338,7 +343,7 @@ public Map loadLatestTweets(@RequestParam String code, @RequestPa try { result = taggerService.loadLatestTweets(code, constraints); } catch (Exception e) { - logger.error(elog.toStringException(e)); + logger.error("Error in loadLatestTweets for code : " + code + " and constraints : " + constraints); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } return getUIWrapper(result,true); @@ -364,7 +369,6 @@ public Map getPublicFlagStatus() { } } catch (Exception e) { logger.error("Unable to fetch list of running collections from DB"); - logger.error(elog.toStringException(e)); } return null; } @@ -387,7 +391,6 @@ public Map getCollectionPublicFlagStatus(@QueryParam("channelCo } } catch (Exception e) { logger.error("Unable to fetch list of running collections from DB"); - logger.error(elog.toStringException(e)); } return null; } @@ -411,7 +414,6 @@ public Map findTotalCount(final String collectionCode) throws E return result; } catch (Exception e) { logger.error("Unable to fetch total count of downloaded documents for collection = " + collectionCode); - logger.error(elog.toStringException(e)); } return null; } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java index 577d16b1e..f002c5e7a 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java @@ -24,7 +24,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.values.DownloadType; import qa.qcri.aidr.dbmanager.dto.taggerapi.TrainingDataDTO; import qa.qcri.aidr.manager.dto.CrisisRequest; @@ -59,7 +58,6 @@ public class TaggerController extends BaseController { private Logger logger = Logger.getLogger(TaggerController.class); - private ErrorLog elog = new ErrorLog(); @Autowired private TaggerService taggerService; @@ -216,7 +214,7 @@ public Map getModelsForCrisis(Integer id) { logger.info("cast long value: " + value); result.get(i).setTrainingExamples(value.longValue()); } catch (Exception e) { - logger.error(elog.toStringException(e)); + logger.error("Error in getModelsForCrisis for id : " + id); } } logger.info("For model family id: " + result.get(i).getModelFamilyID() + ", set human labeled count = " + result.get(i).getTrainingExamples()); @@ -225,7 +223,7 @@ public Map getModelsForCrisis(Integer id) { //return getUIWrapper(taggerService.getModelsForCrisis(id), true); return getUIWrapper(result, true); } catch (AidrException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in getModelsForCrisis for id : " + id, e); return getUIWrapper(false, e.getMessage()); } } @@ -448,16 +446,12 @@ public Map getTrainingDataCountByModelIdAndCrisisId( response = taggerService.getTrainingDataByModelIdAndCrisisId(modelFamilyId, crisisId, start, limit, "", ""); logger.info("received response back: " + response); } catch (AidrException e) { - logger.error(elog.toStringException(e)); + logger.error("Error in getTrainingDataCountByModelIdAndCrisisId for crisisId : " + crisisId + " and modelFamilyId : " + modelFamilyId, e); return getUIWrapper(new Integer(0), false); } Integer total = 0; if (response != null && !response.isEmpty()) { - try { - total = response.get(0).getTotalRows(); - } catch (Exception e) { - logger.error(elog.toStringException(e)); - } + total = response.get(0).getTotalRows(); } logger.info("Returning for crisis ID " + crisisId + ", model family ID " + modelFamilyId + ", human labeled count = " + total); return getUIWrapper(total, true); diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectionAPI.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectionAPI.java index 151f87f63..d8d134db9 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectionAPI.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectionAPI.java @@ -6,28 +6,26 @@ package qa.qcri.aidr.persister.api; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.persister.collction.RedisCollectionPersister; import qa.qcri.aidr.utils.GenericCache; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - @Path("collectionPersister") public class Persister4CollectionAPI { private static Logger logger = Logger.getLogger(Persister4CollectionAPI.class.getName()); - private static ErrorLog elog = new ErrorLog(); @GET @Path("/start") @@ -50,7 +48,6 @@ public Response startPersister(@QueryParam("channel_provider") String provider, } } catch (Exception ex) { logger.error(code + ": Failed to start persister"); - logger.error(elog.toStringException(ex)); response = "Failed to start persister " + ex.getMessage(); } return Response.ok(response).build(); @@ -71,7 +68,6 @@ public Response stopPersister(@QueryParam("collection_code") String code) { return Response.ok(response).build(); } catch (InterruptedException ex) { logger.error(code + ": Failed to stop persister"); - logger.error(elog.toStringException(ex)); } response = "Unable to locate a running persister with the given collection code:[" + code + "]"; return Response.ok(response).build(); diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectorAPI.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectorAPI.java index c21028bdb..ba6518f63 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectorAPI.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectorAPI.java @@ -13,23 +13,22 @@ import java.net.UnknownHostException; import java.util.Map; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.UriInfo; import javax.ws.rs.Consumes; import javax.ws.rs.DefaultValue; -import javax.ws.rs.Path; import javax.ws.rs.GET; +import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; import net.minidev.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.values.DownloadType; import qa.qcri.aidr.persister.collector.RedisCollectorPersister; import qa.qcri.aidr.utils.DownloadJsonType; @@ -44,7 +43,6 @@ public class Persister4CollectorAPI { private static Logger logger = Logger.getLogger(Persister4CollectorAPI.class.getName()); - private static ErrorLog elog = new ErrorLog(); @Context private UriInfo context; @@ -75,7 +73,6 @@ public Response startPersister(@QueryParam("file") String fileLocation, @QueryPa }catch (Exception ex) { //Logger.getLogger(Persister4CollectorAPI.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + ": Failed to start persister"); - logger.error(elog.toStringException(ex)); } return Response.ok(response).build(); } @@ -98,7 +95,6 @@ public Response stopPersister(@QueryParam("collectionCode") String collectionCod } catch (InterruptedException ex) { //Logger.getLogger(Persister4CollectorAPI.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + ": Failed to stop persister"); - logger.error(elog.toStringException(ex)); } } response = "Unable to locate a running persister with the given collection code:[" + collectionCode + "]"; diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4TaggerAPI.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4TaggerAPI.java index aa199cad2..3d797f116 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4TaggerAPI.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4TaggerAPI.java @@ -33,7 +33,6 @@ import qa.qcri.aidr.common.filter.DeserializeFilters; import qa.qcri.aidr.common.filter.JsonQueryList; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.values.DownloadType; import qa.qcri.aidr.persister.tagger.RedisTaggerPersister; import qa.qcri.aidr.utils.ClassifiedTweet; @@ -49,7 +48,6 @@ public class Persister4TaggerAPI { private static Logger logger = Logger.getLogger(Persister4TaggerAPI.class.getName()); - private static ErrorLog elog = new ErrorLog(); @Context private UriInfo context; @@ -80,7 +78,6 @@ public Response startPersister(@QueryParam("file") String fileLocation, @QueryPa } } catch (Exception ex) { logger.error(collectionCode + ": failed to start persister"); - logger.error(elog.toStringException(ex)); } return Response.ok(response).build(); } @@ -104,7 +101,6 @@ public Response stopPersister(@QueryParam("collectionCode") String collectionCod } catch (InterruptedException ex) { //Logger.getLogger(Persister4TaggerAPI.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + ": failed to stop persister"); - logger.error(elog.toStringException(ex)); } } response = "Unable to locate a running tagger persister with the given collection code:[" + collectionCode + "]"; @@ -140,7 +136,6 @@ public Response generateCSVFromLastestJSON(@QueryParam("collectionCode") String return Response.ok(obj.toJSONString()).build(); } catch (Exception e) { logger.error("Unable to return result "); - logger.error(elog.toStringException(e)); obj.putAll(ResultStatus.getUIWrapper(collectionCode, null, fileName, false)); return Response.ok(obj.toJSONString()).build(); diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/CollectionSubscriber.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/CollectionSubscriber.java index 336920b39..1c1d3c9ab 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/CollectionSubscriber.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/CollectionSubscriber.java @@ -6,26 +6,27 @@ package qa.qcri.aidr.persister.collction; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.ConcurrentHashMap; + import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.redis.LoadShedder; import qa.qcri.aidr.io.FileSystemOperations; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; import redis.clients.jedis.JedisPubSub; -import java.io.*; -import java.nio.charset.Charset; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.concurrent.ConcurrentHashMap; - public class CollectionSubscriber extends JedisPubSub { private static Logger logger = Logger.getLogger(CollectionSubscriber.class.getName()); - private static ErrorLog elog = new ErrorLog(); private String persisterDir; private String collectionDir; @@ -95,7 +96,6 @@ private void createNewFile() { } } catch (IOException ex) { logger.error(collectionCode + " error in creating new file at location " + collectionDir); - logger.error(elog.toStringException(ex)); } } @@ -120,7 +120,6 @@ private void createBufferWriter() { out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(this.file, true)), Integer.parseInt(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_FILE_WRITER_BUFFER_SIZE))); } catch (IOException ex) { logger.error(collectionCode + "Error in creating Buffered writer"); - logger.error(elog.toStringException(ex)); } } @@ -132,7 +131,6 @@ private void writeToFile(String message) { isTimeToCreateNewFile(); } catch (IOException ex) { logger.error(collectionCode + "Error in writing to file"); - logger.error(elog.toStringException(ex)); } } @@ -154,7 +152,6 @@ public void closeFileWriting() { } } catch (IOException ex) { logger.error(collectionCode + "Error in closing file writer"); - logger.error(elog.toStringException(ex)); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/RedisCollectionPersister.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/RedisCollectionPersister.java index abd9a325f..1f7ad8cf2 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/RedisCollectionPersister.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/RedisCollectionPersister.java @@ -3,7 +3,6 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.redis.JedisConnectionPool; import redis.clients.jedis.Jedis; @@ -14,7 +13,6 @@ public class RedisCollectionPersister implements Runnable { private static Logger logger = Logger.getLogger(RedisCollectionPersister.class.getName()); - private static ErrorLog elog = new ErrorLog(); String fileName; Thread t; boolean suspendFlag; @@ -38,7 +36,6 @@ public RedisCollectionPersister(String fileName, String channel, String collecti } catch (Exception e) { // TODO Auto-generated catch block logger.error(collectionCode + ": Error in subscribing to Redis"); - logger.error(elog.toStringException(e)); connObject.close(subscriberJedis); subscriberJedis = null; @@ -68,7 +65,6 @@ public void run() { Thread.sleep(200); } catch (InterruptedException ex) { logger.warn(collectionCode + " error in closing Redis connection"); - logger.warn(elog.toStringException(ex)); } } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/CollectorSubscriber.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/CollectorSubscriber.java index 6d4c841d4..235aedf99 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/CollectorSubscriber.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/CollectorSubscriber.java @@ -6,31 +6,27 @@ package qa.qcri.aidr.persister.collector; +import java.io.BufferedWriter; import java.io.File; -import java.io.OutputStreamWriter; import java.io.FileOutputStream; -import java.io.BufferedWriter; import java.io.IOException; -import java.nio.charset.Charset; +import java.io.OutputStreamWriter; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.concurrent.ConcurrentHashMap; -import redis.clients.jedis.JedisPubSub; - import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.redis.LoadShedder; import qa.qcri.aidr.io.FileSystemOperations; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; +import redis.clients.jedis.JedisPubSub; public class CollectorSubscriber extends JedisPubSub { private static Logger logger = Logger.getLogger(CollectorSubscriber.class.getName()); - private static ErrorLog elog = new ErrorLog(); private String persisterDir; private String collectionDir; @@ -101,7 +97,6 @@ private void createNewFile() { } } catch (IOException ex) { logger.error(collectionCode + " error in creating new file at location " + collectionDir); - logger.error(elog.toStringException(ex)); } } @@ -127,7 +122,6 @@ private void createBufferWriter() { } catch (IOException ex) { //Logger.getLogger(CollectorSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in creating Buffered writer"); - logger.error(elog.toStringException(ex)); } } @@ -140,7 +134,6 @@ private void writeToFile(String message) { } catch (IOException ex) { //Logger.getLogger(CollectorSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in writing to file"); - logger.error(elog.toStringException(ex)); } } @@ -161,7 +154,6 @@ public void closeFileWriting() { } catch (IOException ex) { //Logger.getLogger(CollectorSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in closing file writer"); - logger.error(elog.toStringException(ex)); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/RedisCollectorPersister.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/RedisCollectorPersister.java index 09e7e1bdf..d13c2b0de 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/RedisCollectorPersister.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/RedisCollectorPersister.java @@ -7,9 +7,7 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.redis.JedisConnectionPool; -import qa.qcri.aidr.utils.GenericCache; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; import redis.clients.jedis.Jedis; @@ -18,7 +16,6 @@ public class RedisCollectorPersister implements Runnable { private static Logger logger = Logger.getLogger(RedisCollectorPersister.class.getName()); - private static ErrorLog elog = new ErrorLog(); String fileName; Thread t; @@ -45,7 +42,6 @@ public RedisCollectorPersister(String fileName, String collectionCode) throws In } catch (Exception e) { // TODO Auto-generated catch block logger.error(collectionCode + ": Error in subscribing to Redis"); - logger.error(elog.toStringException(e)); connObject.close(subscriberJedis); subscriberJedis = null; @@ -78,7 +74,6 @@ public void run() { } catch (InterruptedException ex) { //Logger.getLogger(RedisCollectorPersister.class.getName()).log(Level.SEVERE, null, ex); logger.warn(collectionCode + " error in closing Redis connection"); - logger.warn(elog.toStringException(ex)); } } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/RedisTaggerPersister.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/RedisTaggerPersister.java index 48298d18a..e128530c2 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/RedisTaggerPersister.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/RedisTaggerPersister.java @@ -7,10 +7,7 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.persister.collector.*; import qa.qcri.aidr.redis.JedisConnectionPool; -import qa.qcri.aidr.utils.GenericCache; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; import redis.clients.jedis.Jedis; @@ -19,7 +16,6 @@ public class RedisTaggerPersister implements Runnable { private static Logger logger = Logger.getLogger(RedisTaggerPersister.class.getName()); - private static ErrorLog elog = new ErrorLog(); String fileName; Thread t; @@ -44,8 +40,6 @@ public RedisTaggerPersister(String fileName, String collectionCode) throws Inter subscriber = new TaggerSubscriber(fileName, collectionCode); } catch (Exception e) { logger.error(collectionCode + " error in subscribing to Redis"); - logger.error(elog.toStringException(e)); - connObject.close(subscriberJedis); subscriberJedis = null; subscriber = null; @@ -75,7 +69,6 @@ public void run() { Thread.sleep(200); } catch (InterruptedException ex) { logger.warn(collectionCode + ": Error in closing Redis connection"); - logger.warn(elog.toStringException(ex)); } } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/TaggerSubscriber.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/TaggerSubscriber.java index 3517e2019..26e99d977 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/TaggerSubscriber.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/TaggerSubscriber.java @@ -6,19 +6,11 @@ package qa.qcri.aidr.persister.tagger; -import qa.qcri.aidr.persister.collector.*; - +import java.io.BufferedWriter; import java.io.File; -import java.io.OutputStreamWriter; import java.io.FileOutputStream; -import java.io.BufferedWriter; import java.io.IOException; -import java.nio.charset.Charset; - -import qa.qcri.aidr.utils.ClassifiedTweet; -import qa.qcri.aidr.utils.PersisterConfigurationProperty; -import qa.qcri.aidr.utils.PersisterConfigurator; - +import java.io.OutputStreamWriter; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; @@ -26,17 +18,18 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.utils.JsonDeserializer; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.common.redis.LoadShedder; import qa.qcri.aidr.io.FileSystemOperations; +import qa.qcri.aidr.utils.ClassifiedTweet; +import qa.qcri.aidr.utils.JsonDeserializer; +import qa.qcri.aidr.utils.PersisterConfigurationProperty; +import qa.qcri.aidr.utils.PersisterConfigurator; import redis.clients.jedis.JedisPubSub; public class TaggerSubscriber extends JedisPubSub { private static Logger logger = Logger.getLogger(TaggerSubscriber.class.getName()); - private static ErrorLog elog = new ErrorLog(); private String persisterDir; private String collectionDir; @@ -122,7 +115,6 @@ private void createNewFile() { } catch (IOException ex) { //Logger.getLogger(TaggerSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + " error in creating new file at location " + collectionDir); - logger.error(elog.toStringException(ex)); } } @@ -149,7 +141,6 @@ private void createBufferWriter() { } catch (IOException ex) { //Logger.getLogger(TaggerSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in creating Buffered writer"); - logger.error(elog.toStringException(ex)); } } @@ -162,7 +153,6 @@ private void writeToFile(String message) { } catch (IOException ex) { //Logger.getLogger(TaggerSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in writing to file"); - logger.error(elog.toStringException(ex)); } // Debug code added by koushik @@ -194,7 +184,6 @@ public void closeFileWriting() { } catch (IOException ex) { //Logger.getLogger(TaggerSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in closing file writer"); - logger.error(elog.toStringException(ex)); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/redis/JedisConnectionPool.java b/aidr-persister/src/main/java/qa/qcri/aidr/redis/JedisConnectionPool.java index 4d5b165d7..656e3d65c 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/redis/JedisConnectionPool.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/redis/JedisConnectionPool.java @@ -7,7 +7,6 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; import redis.clients.jedis.Jedis; @@ -19,7 +18,6 @@ public class JedisConnectionPool { private static Logger logger = Logger.getLogger(JedisConnectionPool.class.getName()); - private static ErrorLog elog = new ErrorLog(); static JedisPool jedisPool; @@ -45,7 +43,6 @@ public synchronized Jedis getJedisConnection() { // koushik: removed static return jedisPool.getResource(); } catch (JedisConnectionException e) { logger.error("Could not establish Redis connection. Is the Redis running?"); - logger.error(elog.toStringException(e)); throw e; } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java index 9f528bd94..d776767d1 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java @@ -41,7 +41,6 @@ import qa.qcri.aidr.common.filter.FilterQueryMatcher; import qa.qcri.aidr.common.filter.JsonQueryList; import qa.qcri.aidr.common.filter.NominalLabel; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentDTO; import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentList; import qa.qcri.aidr.io.FileSystemOperations; @@ -57,7 +56,6 @@ public class JsonDeserializer { private static Logger logger = Logger.getLogger(JsonDeserializer.class.getName()); - private static ErrorLog elog = new ErrorLog(); private static final int BUFFER_SIZE = 10 * 1024 * 1024; // buffer size to use for buffered r/w private static final int LIST_BUFFER_SIZE = 50000; @@ -127,10 +125,8 @@ public Map generateJson2TweetIdsCSV(String collectionCode, boole br.close(); } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file = " + fileLocation); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file = " + fileLocation); - logger.error(elog.toStringException(ex)); } } // end for int countToWrite = tweetsList.size(); @@ -148,7 +144,6 @@ public Map generateJson2TweetIdsCSV(String collectionCode, boole beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for csv file write "); - logger.error(elog.toStringException(ex)); } } } @@ -222,10 +217,8 @@ public Map generateClassifiedJson2TweetIdsCSV(String collectionC } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file = " + fileLocation); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file = " + fileLocation); - logger.error(elog.toStringException(ex)); } } // end for int countToWrite = tweetsList.size(); @@ -247,7 +240,6 @@ public Map generateClassifiedJson2TweetIdsCSV(String collectionC writer.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for csv file write "); - logger.error(elog.toStringException(ex)); } } } @@ -348,10 +340,8 @@ public Map generateClassifiedJson2TweetIdsCSVFiltered(final Stri } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file = " + fileLocation); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file = " + fileLocation); - logger.error(elog.toStringException(ex)); } } // end for int countToWrite = tweetsList.size(); @@ -372,7 +362,6 @@ public Map generateClassifiedJson2TweetIdsCSVFiltered(final Stri writer.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for csv file write "); - logger.error(elog.toStringException(ex)); } } } @@ -479,18 +468,15 @@ public int compare(File f1, File f2) { } } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); isCSVGenerated = false; } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file"); - logger.error(elog.toStringException(ex)); } finally { if (beanWriter != null) { try { beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for csv file write "); - logger.error(elog.toStringException(ex)); } } } @@ -588,17 +574,14 @@ public int compare(File f1, File f2) { } } catch (FileNotFoundException ex) { logger.error(collectionCode + ": File not found."); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } finally { if (writer != null) { try { writer.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for csv file write "); - logger.error(elog.toStringException(ex)); } } } @@ -733,17 +716,14 @@ public int compare(File f1, File f2) { } } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } finally { if (writer != null) { try { writer.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for csv file write "); - logger.error(elog.toStringException(ex)); } } } @@ -802,10 +782,8 @@ public int compare(File f1, File f2) { } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } return tweetsList; } @@ -874,10 +852,8 @@ public int compare(File f1, File f2) { } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } return tweetsList; } @@ -1147,10 +1123,8 @@ public int compare(File f1, File f2) { } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } finally { if (beanWriter != null) { try { @@ -1161,7 +1135,6 @@ public int compare(File f1, File f2) { beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for JSON file write "); - logger.error(elog.toStringException(ex)); } } } @@ -1248,10 +1221,8 @@ public int compare(File f1, File f2) { } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } finally { if (beanWriter != null) { try { @@ -1262,7 +1233,6 @@ public int compare(File f1, File f2) { beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for JSON file write "); - logger.error(elog.toStringException(ex)); } } } @@ -1363,13 +1333,10 @@ public int compare(File f1, File f2) { } // end for } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } catch (NullPointerException ex) { logger.error(collectionCode + ": empty list of files to read"); - logger.info(elog.toStringException(ex)); } finally { if (beanWriter != null) { try { @@ -1380,7 +1347,6 @@ public int compare(File f1, File f2) { beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for JSON file write "); - logger.error(elog.toStringException(ex)); } } } @@ -1449,10 +1415,8 @@ public Map generateClassifiedJson2TweetIdsJSON(String collection } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } } // end for if (DownloadJsonType.JSON_OBJECT.equals(jsonType) && !jsonObjectClosed) { @@ -1464,10 +1428,8 @@ public Map generateClassifiedJson2TweetIdsJSON(String collection } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } finally { if (beanWriter != null) { try { @@ -1478,7 +1440,6 @@ public Map generateClassifiedJson2TweetIdsJSON(String collection beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for JSON file write "); - logger.error(elog.toStringException(ex)); } } } @@ -1600,13 +1561,10 @@ public int compare(File f1, File f2) { } // end for } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } catch (NullPointerException ex) { logger.error(collectionCode + ": empty list of files to read"); - logger.info(elog.toStringException(ex)); } finally { if (beanWriter != null) { try { @@ -1618,7 +1576,6 @@ public int compare(File f1, File f2) { beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for JSON file write "); - logger.error(elog.toStringException(ex)); } } } @@ -1715,10 +1672,8 @@ public Map generateClassifiedJson2TweetIdsJSONFiltered( } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } } // end for if (DownloadJsonType.JSON_OBJECT.equals(jsonType) && !jsonObjectClosed) { @@ -1730,10 +1685,8 @@ public Map generateClassifiedJson2TweetIdsJSONFiltered( } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); - logger.error(elog.toStringException(ex)); } catch (IOException ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } finally { if (beanWriter != null) { try { @@ -1744,7 +1697,6 @@ public Map generateClassifiedJson2TweetIdsJSONFiltered( beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for JSON file write "); - logger.error(elog.toStringException(ex)); } } } @@ -1974,7 +1926,6 @@ public Map generateClassifiedList2TweetIdsCSVFiltered(String col writer.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for csv file write "); - logger.error(elog.toStringException(ex)); } } } @@ -2066,7 +2017,6 @@ public String generateClassifiedList2JSON_100K_BasedOnTweetCountFiltered(String } // end for } catch (Exception ex) { logger.error(collectionCode + ": empty list of files to read"); - logger.info(elog.toStringException(ex)); } finally { if (beanWriter != null) { try { @@ -2078,7 +2028,6 @@ public String generateClassifiedList2JSON_100K_BasedOnTweetCountFiltered(String beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for JSON file write "); - logger.error(elog.toStringException(ex)); } } } @@ -2159,7 +2108,6 @@ public Map generateClassifiedList2TweetIdsJSONFiltered(String co beanWriter.close(); } catch (Exception ex) { logger.error(collectionCode + ": IO Exception for file read"); - logger.error(elog.toStringException(ex)); } finally { if (beanWriter != null) { try { @@ -2170,7 +2118,6 @@ public Map generateClassifiedList2TweetIdsJSONFiltered(String co beanWriter.close(); } catch (IOException ex) { logger.error(collectionCode + ": IOException for JSON file write "); - logger.error(elog.toStringException(ex)); } } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ResultStatus.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ResultStatus.java index 0d26a3e53..eddcee5f7 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ResultStatus.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ResultStatus.java @@ -13,12 +13,10 @@ import org.glassfish.jersey.jackson.JacksonFeature; import qa.qcri.aidr.common.code.ResponseWrapper; -import qa.qcri.aidr.common.logging.ErrorLog; public class ResultStatus extends ResponseWrapper { private static Logger logger = Logger.getLogger(ResultStatus.class); - private static ErrorLog elog = new ErrorLog(); @Deprecated @SuppressWarnings({ "unused", "unchecked" }) @@ -44,7 +42,6 @@ public static Integer getTotalDownloadedCount(final String collectionCode) { } } catch (Exception e) { logger.error("Error in querying manager for running collections: " + clientResponse); - logger.error(elog.toStringException(e)); } return null; } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/NominalAttributeResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/NominalAttributeResource.java index 21af01861..689edc411 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/NominalAttributeResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/NominalAttributeResource.java @@ -6,26 +6,21 @@ import java.util.List; -import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.predictui.util.ResponseWrapper; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; - import javax.ejb.EJB; import javax.ejb.Stateless; import javax.ws.rs.Consumes; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.UriInfo; -import javax.ws.rs.Path; +import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; -import javax.ws.rs.DELETE; +import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; import org.apache.log4j.Logger; @@ -33,6 +28,9 @@ import qa.qcri.aidr.dbmanager.dto.CrisisAttributesDTO; import qa.qcri.aidr.dbmanager.dto.NominalAttributeDTO; import qa.qcri.aidr.predictui.facade.NominalAttributeFacade; +import qa.qcri.aidr.predictui.util.ResponseWrapper; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; /** * REST Web Service diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisNotifier.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisNotifier.java index cf8aaf6b5..1bee42118 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisNotifier.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisNotifier.java @@ -2,7 +2,6 @@ import org.apache.log4j.Logger; -import qa.qcri.aidr.common.logging.ErrorLog; import redis.clients.jedis.Jedis; /** diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentServiceImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentServiceImpl.java index 012fce48f..31ea29e34 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentServiceImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentServiceImpl.java @@ -8,7 +8,6 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import qa.qcri.aidr.common.logging.ErrorLog; import qa.qcri.aidr.dbmanager.dto.DocumentDTO; import qa.qcri.aidr.dbmanager.entities.task.Document; import qa.qcri.aidr.task.common.TrainingDataFetchType; diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java index 8ca16cdec..e75663016 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java @@ -21,6 +21,9 @@ import qa.qcri.aidr.trainer.api.template.PybossaTemplate; import qa.qcri.aidr.trainer.api.template.TaskAnswerResponse; +import qa.qcri.aidr.dbmanager.dto.DocumentDTO; +import qa.qcri.aidr.dbmanager.dto.TaskAnswerDTO; + /** * Created with IntelliJ IDEA. * User: jilucas, koushik diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAssignmentServiceImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAssignmentServiceImpl.java index 5999c223f..529aee81f 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAssignmentServiceImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAssignmentServiceImpl.java @@ -1,5 +1,7 @@ package qa.qcri.aidr.trainer.api.service.impl; +import java.util.List; + import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -7,14 +9,11 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; -import qa.qcri.aidr.dbmanager.dto.DocumentDTO; import qa.qcri.aidr.task.ejb.TaskManagerRemote; import qa.qcri.aidr.trainer.api.dao.UsersDao; import qa.qcri.aidr.trainer.api.entity.Users; import qa.qcri.aidr.trainer.api.service.TaskAssignmentService; - -import java.util.List; - +import qa.qcri.aidr.dbmanager.dto.DocumentDTO; @Service("taskAssignmentService") @Transactional(readOnly = true) diff --git a/deploy.sh b/deploy.sh index c0be19217..f5a53fce2 100755 --- a/deploy.sh +++ b/deploy.sh @@ -8,7 +8,7 @@ if [ "$MODE" == "" ] then echo "ERROR: Missing argument MODE. Please provide a value {deploy, undeploy, undeploy-deploy}." exit -elif [ "$MODE" != "undeploy" ] && [ "$MODE" != "deploy" ] && [ "$MODE" != "undeploy-deploy" ] && [ "$MODE" != "redeploy" ] +elif [ "$MODE" != "undeploy" ] && [ "$MODE" != "deploy" ] && [ "$MODE" != "undeploy-deploy" ] then echo "ERROR: Incorrect first argument provided. Please use 'deploy', 'undeploy' or 'undeploy-deploy'." exit @@ -22,8 +22,8 @@ fi echo "Step 1: Setting up environment variables." # Setting environment variables. -GLASSFISH_HOME=/opt/glassfish4 -AIDR_HOME=/home/meghna/Documents/GitHub/DEV +GLASSFISH_HOME=C:/Users/Latika/Desktop/Metacube/Installed/glassfish4 +AIDR_HOME=C:/Users/Latika/Desktop/Metacube/QCRI/AIDR/dev_new AIDR_GLASSFISH_DOMAIN=domain1 MY_SQL_USERNAME=root AIDR_ANALYSIS_CONNECTION_POOL=AIDR_ANALYSIS_CONNECTION_POOL @@ -63,11 +63,6 @@ bin/asadmin undeploy AIDROutput bin/asadmin undeploy AIDRAnalytics bin/asadmin undeploy AIDRTrainerAPI bin/asadmin undeploy AIDRFetchManager - -echo "Stopping Application AIDRTagger." -PID=$(ps -eo pid,cmd | grep [q]a.qcri.aidr.predict.Controller | awk '{print $1}') -kill -9 $PID - echo "Done." # stop the glass fish domain. @@ -99,43 +94,20 @@ bin/asadmin deploy --contextroot=AIDRDBManager --name=AIDRDBManager $AIDR_HOME/a bin/asadmin deploy --contextroot=AIDRTaskManager --name=AIDRTaskManager $AIDR_HOME/aidr-task-manager/target/aidr-task-manager-ear-1.0.ear bin/asadmin deploy --contextroot=AIDRPersister --name=AIDRPersister $AIDR_HOME/aidr-persister/target/aidr-persister-1.0.war bin/asadmin deploy --contextroot=AIDRCollector --name=AIDRCollector $AIDR_HOME/aidr-collector/target/aidr-collector-1.0.war + +echo "Starting Application AIDRTagger." +#cd $AIDR_HOME/aidr-tagger/target +#java -Xmx4048m -cp "$GLASSFISH_HOME/glassfish/lib/gf-client.jar;aidr-tagger-1.0-jar-with-dependencies.jar;libs/*" qa.qcri.aidr.predict.Controller + +cd $GLASSFISH_HOME bin/asadmin deploy --contextroot=AIDRTaggerAPI --name=AIDRTaggerAPI $AIDR_HOME/aidr-tagger-api/target/aidr-tagger-api-1.0.war bin/asadmin deploy --contextroot=AIDROutput --name=AIDROutput $AIDR_HOME/aidr-output/target/aidr-output-1.0.war bin/asadmin deploy --contextroot=AIDRAnalytics --name=AIDRAnalytics $AIDR_HOME/aidr-analytics/target/aidr-analytics-1.0.war bin/asadmin deploy --contextroot=AIDRTrainerAPI --name=AIDRTrainerAPI $AIDR_HOME/aidr-trainer-api/target/aidr-trainer-api.war - bin/asadmin set configs.config.server-config.cdi-service.enable-implicit-cdi=false bin/asadmin deploy --properties implicitCdiEnabled=false --contextroot=AIDRFetchManager --name=AIDRFetchManager $AIDR_HOME/aidr-manager/target/aidr-manager.war bin/asadmin set configs.config.server-config.cdi-service.enable-implicit-cdi=true -echo "Starting Application AIDRTagger." -cd $AIDR_HOME/aidr-tagger/target -nohup java -Xmx2048m -cp $GLASSFISH_HOME/glassfish/lib/gf-client.jar:aidr-tagger-1.0-jar-with-dependencies.jar:libs/* qa.qcri.aidr.predict.Controller & -fi - -if [ "$MODE" == "redeploy" ] -then - -# Deploying separate modules. First undeploying if already deployed and deploying again. -bin/asadmin redeploy --keepstate=true --contextroot=AIDRDBManager --name=AIDRDBManager $AIDR_HOME/aidr-db-manager/target/aidr-db-manager-ear-1.0.ear -bin/asadmin redeploy --keepstate=true --contextroot=AIDRTaskManager --name=AIDRTaskManager $AIDR_HOME/aidr-task-manager/target/aidr-task-manager-ear-1.0.ear -bin/asadmin redeploy --keepstate=true --contextroot=AIDRPersister --name=AIDRPersister $AIDR_HOME/aidr-persister/target/aidr-persister-1.0.war -bin/asadmin redeploy --keepstate=true --contextroot=AIDRCollector --name=AIDRCollector $AIDR_HOME/aidr-collector/target/aidr-collector-1.0.war -bin/asadmin redeploy --keepstate=true --contextroot=AIDRTaggerAPI --name=AIDRTaggerAPI $AIDR_HOME/aidr-tagger-api/target/aidr-tagger-api-1.0.war -bin/asadmin redeploy --keepstate=true --contextroot=AIDROutput --name=AIDROutput $AIDR_HOME/aidr-output/target/aidr-output-1.0.war -bin/asadmin redeploy --keepstate=true --contextroot=AIDRAnalytics --name=AIDRAnalytics $AIDR_HOME/aidr-analytics/target/aidr-analytics-1.0.war -bin/asadmin redeploy --keepstate=true --contextroot=AIDRTrainerAPI --name=AIDRTrainerAPI $AIDR_HOME/aidr-trainer-api/target/aidr-trainer-api.war -bin/asadmin set configs.config.server-config.cdi-service.enable-implicit-cdi=false -bin/asadmin redeploy --properties implicitCdiEnabled=false --contextroot=AIDRFetchManager --name=AIDRFetchManager $AIDR_HOME/aidr-manager/target/aidr-manager.war -bin/asadmin set configs.config.server-config.cdi-service.enable-implicit-cdi=true - -echo "Stopping Application AIDRTagger." -PID=$(ps -eo pid,cmd | grep [q]cri.aidr.predict.Controller | awk '{print $1}') -kill -9 $PID - -echo "Starting Application AIDRTagger." -cd $AIDR_HOME/aidr-tagger -nohup java -Xmx2048m -cp $GLASSFISH_HOME/glassfish/lib/gf-client.jar:target/aidr-tagger-1.0-jar-with-dependencies.jar:lib-non-maven/* qa.qcri.aidr.predict.Controller & fi if [ "$RUN_DDL" == "deploy_db" ] diff --git a/profiles/dev/config.properties b/profiles/dev/config.properties index 871dcc04d..f92e30fd2 100644 --- a/profiles/dev/config.properties +++ b/profiles/dev/config.properties @@ -23,14 +23,14 @@ SQL_SERVER_TYPE=mysql HIBERNATE_DIALECT=org.hibernate.dialect.MySQLDialect #Server URL is used for persister download URL and twitter callback in aidr-manager -AIDR_SERVER_URL= +AIDR_SERVER_URL=localhost:8080 COLLECTOR_HOST=localhost -COLLECTOR_PORT=8084 +COLLECTOR_PORT=8080 COLLECTOR_WEB_APP_CONTEXT=AIDRCollector PERSISTER_HOST=localhost -PERSISTER_PORT=8084 +PERSISTER_PORT=8080 PERSISTER_WEB_APP_CONTEXT=AIDRPersister MANAGER_HOST=localhost @@ -38,20 +38,20 @@ MANAGER_PORT=8080 MANAGER_WEB_APP_CONTEXT=AIDRFetchManager DB_MANAGER_HOST=localhost -DB_MANAGER_PORT=8084 +DB_MANAGER_PORT=8080 DB_MANAGER_HBM2DDL=update DB_MANAGER_JNDI=JNDI/aidr_db_manager OUTPUT_API_HOST=localhost -OUTPUT_API_PORT=8084 +OUTPUT_API_PORT=8080 OUTPUT_API_WEB_APP_CONTEXT=AIDROutput TAGGER_API_HOST=localhost -TAGGER_API_PORT=8084 +TAGGER_API_PORT=8080 TAGGER_API_WEB_APP_CONTEXT=AIDRTaggerAPI TRAINER_API_HOST=localhost -TRAINER_API_PORT=8084 +TRAINER_API_PORT=8080 TRAINER_API_WEB_APP_CONTEXT=AIDRTrainerAPI MANAGER_DB_NAME=aidr_fetch_manager @@ -76,23 +76,23 @@ ANALYTICS_DB_PASSWORD=aidr_admin ANALYTICS_DB_HBM2DDL=update ANALYTICS_JNDI=JNDI/aidr_analysis -DEFAULT_PERSISTER_FILE_PATH=/sc/aidr/data/persister/ +DEFAULT_PERSISTER_FILE_PATH=C:/Users/Latika/Desktop/Metacube/QCRI/temp/sc/aidr/data/persister/ # Twitter specific configurations -twitter.consumerKey= -twitter.consumerSecret= +twitter.consumerKey=hEIRfBfxNt8HlnfmRGkwrqIGb +twitter.consumerSecret=swVbpPSxJsSFP0tifvQ1vVxhMDY3bpfdZT8nHOOfC8EIKMzTuD # root path of aidr-tagger. Path to the config.properties file is derived # through this. -TAGGER_HOME=/home/aidr-tagger +TAGGER_HOME=C:/Users/Latika/Desktop/Metacube/QCRI/AIDR/dev_new/aidr-tagger # Path to a directory where trained models are stored # this location is used by aidr-tagger.jar -MODEL_STORAGE_PATH=model +MODEL_STORAGE_PATH=C:/Users/Latika/Desktop/Metacube/QCRI/temp/model #Comma separated list of recipient emails #In case of fatal errors in the system, emails will be sent -RECIPIENT_EMAILS= +RECIPIENT_EMAILS=latika.bhurani@metacube.com,kushalkant.goyal@metacube.com ############################# # OPTIONAL configuration @@ -121,13 +121,12 @@ REMOTE_TASK_MANAGER_JNDI_PREFIX=java:global/AIDRTaskManager/aidr-task-manager-ej REMOTE_DB_MANAGER_JNDI_PREFIX=java:global/AIDRDBManager/aidr-db-manager-ejb-1.0 #Email configuration -SENDER_EMAIL= -SENDER_PASS= +SENDER_EMAIL=aidr.qcri@gmail.com +SENDER_PASS=qcrisc2015 SMTP_HOST=smtp.gmail.com SMTP_PORT=587 -MAIL_SUBJECT= -MAIL_BODY= - +MAIL_SUBJECT=Twitter Exception in collection: +MAIL_BODY= Unable to start the collection within the max retry attempts. Caused by: ############################# # FIXED configuration # Values should not be changed, as they may @@ -142,5 +141,5 @@ STATUS_CODE_WARNING=WARNING STATUS_CODE_FAILED=FAILED # Log settings to be used in all modules -LOG_LEVEL=INFO -LOG_LOCATION=/var/log/aidr +LOG_LEVEL=DEBUG +LOG_LOCATION=C:/Users/Latika/Desktop/Metacube/QCRI/temp/var/log/aidr \ No newline at end of file From f193f66e9865a790e06b33f6f22928549bf776c8 Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Thu, 30 Jul 2015 19:40:07 +0530 Subject: [PATCH 22/44] Reverting changes for deploy.sh and config.properties --- deploy.sh | 46 +++++++++++++++++++++++++++------- profiles/dev/config.properties | 39 ++++++++++++++-------------- 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/deploy.sh b/deploy.sh index f5a53fce2..c0be19217 100755 --- a/deploy.sh +++ b/deploy.sh @@ -8,7 +8,7 @@ if [ "$MODE" == "" ] then echo "ERROR: Missing argument MODE. Please provide a value {deploy, undeploy, undeploy-deploy}." exit -elif [ "$MODE" != "undeploy" ] && [ "$MODE" != "deploy" ] && [ "$MODE" != "undeploy-deploy" ] +elif [ "$MODE" != "undeploy" ] && [ "$MODE" != "deploy" ] && [ "$MODE" != "undeploy-deploy" ] && [ "$MODE" != "redeploy" ] then echo "ERROR: Incorrect first argument provided. Please use 'deploy', 'undeploy' or 'undeploy-deploy'." exit @@ -22,8 +22,8 @@ fi echo "Step 1: Setting up environment variables." # Setting environment variables. -GLASSFISH_HOME=C:/Users/Latika/Desktop/Metacube/Installed/glassfish4 -AIDR_HOME=C:/Users/Latika/Desktop/Metacube/QCRI/AIDR/dev_new +GLASSFISH_HOME=/opt/glassfish4 +AIDR_HOME=/home/meghna/Documents/GitHub/DEV AIDR_GLASSFISH_DOMAIN=domain1 MY_SQL_USERNAME=root AIDR_ANALYSIS_CONNECTION_POOL=AIDR_ANALYSIS_CONNECTION_POOL @@ -63,6 +63,11 @@ bin/asadmin undeploy AIDROutput bin/asadmin undeploy AIDRAnalytics bin/asadmin undeploy AIDRTrainerAPI bin/asadmin undeploy AIDRFetchManager + +echo "Stopping Application AIDRTagger." +PID=$(ps -eo pid,cmd | grep [q]a.qcri.aidr.predict.Controller | awk '{print $1}') +kill -9 $PID + echo "Done." # stop the glass fish domain. @@ -94,20 +99,43 @@ bin/asadmin deploy --contextroot=AIDRDBManager --name=AIDRDBManager $AIDR_HOME/a bin/asadmin deploy --contextroot=AIDRTaskManager --name=AIDRTaskManager $AIDR_HOME/aidr-task-manager/target/aidr-task-manager-ear-1.0.ear bin/asadmin deploy --contextroot=AIDRPersister --name=AIDRPersister $AIDR_HOME/aidr-persister/target/aidr-persister-1.0.war bin/asadmin deploy --contextroot=AIDRCollector --name=AIDRCollector $AIDR_HOME/aidr-collector/target/aidr-collector-1.0.war - -echo "Starting Application AIDRTagger." -#cd $AIDR_HOME/aidr-tagger/target -#java -Xmx4048m -cp "$GLASSFISH_HOME/glassfish/lib/gf-client.jar;aidr-tagger-1.0-jar-with-dependencies.jar;libs/*" qa.qcri.aidr.predict.Controller - -cd $GLASSFISH_HOME bin/asadmin deploy --contextroot=AIDRTaggerAPI --name=AIDRTaggerAPI $AIDR_HOME/aidr-tagger-api/target/aidr-tagger-api-1.0.war bin/asadmin deploy --contextroot=AIDROutput --name=AIDROutput $AIDR_HOME/aidr-output/target/aidr-output-1.0.war bin/asadmin deploy --contextroot=AIDRAnalytics --name=AIDRAnalytics $AIDR_HOME/aidr-analytics/target/aidr-analytics-1.0.war bin/asadmin deploy --contextroot=AIDRTrainerAPI --name=AIDRTrainerAPI $AIDR_HOME/aidr-trainer-api/target/aidr-trainer-api.war + bin/asadmin set configs.config.server-config.cdi-service.enable-implicit-cdi=false bin/asadmin deploy --properties implicitCdiEnabled=false --contextroot=AIDRFetchManager --name=AIDRFetchManager $AIDR_HOME/aidr-manager/target/aidr-manager.war bin/asadmin set configs.config.server-config.cdi-service.enable-implicit-cdi=true +echo "Starting Application AIDRTagger." +cd $AIDR_HOME/aidr-tagger/target +nohup java -Xmx2048m -cp $GLASSFISH_HOME/glassfish/lib/gf-client.jar:aidr-tagger-1.0-jar-with-dependencies.jar:libs/* qa.qcri.aidr.predict.Controller & +fi + +if [ "$MODE" == "redeploy" ] +then + +# Deploying separate modules. First undeploying if already deployed and deploying again. +bin/asadmin redeploy --keepstate=true --contextroot=AIDRDBManager --name=AIDRDBManager $AIDR_HOME/aidr-db-manager/target/aidr-db-manager-ear-1.0.ear +bin/asadmin redeploy --keepstate=true --contextroot=AIDRTaskManager --name=AIDRTaskManager $AIDR_HOME/aidr-task-manager/target/aidr-task-manager-ear-1.0.ear +bin/asadmin redeploy --keepstate=true --contextroot=AIDRPersister --name=AIDRPersister $AIDR_HOME/aidr-persister/target/aidr-persister-1.0.war +bin/asadmin redeploy --keepstate=true --contextroot=AIDRCollector --name=AIDRCollector $AIDR_HOME/aidr-collector/target/aidr-collector-1.0.war +bin/asadmin redeploy --keepstate=true --contextroot=AIDRTaggerAPI --name=AIDRTaggerAPI $AIDR_HOME/aidr-tagger-api/target/aidr-tagger-api-1.0.war +bin/asadmin redeploy --keepstate=true --contextroot=AIDROutput --name=AIDROutput $AIDR_HOME/aidr-output/target/aidr-output-1.0.war +bin/asadmin redeploy --keepstate=true --contextroot=AIDRAnalytics --name=AIDRAnalytics $AIDR_HOME/aidr-analytics/target/aidr-analytics-1.0.war +bin/asadmin redeploy --keepstate=true --contextroot=AIDRTrainerAPI --name=AIDRTrainerAPI $AIDR_HOME/aidr-trainer-api/target/aidr-trainer-api.war +bin/asadmin set configs.config.server-config.cdi-service.enable-implicit-cdi=false +bin/asadmin redeploy --properties implicitCdiEnabled=false --contextroot=AIDRFetchManager --name=AIDRFetchManager $AIDR_HOME/aidr-manager/target/aidr-manager.war +bin/asadmin set configs.config.server-config.cdi-service.enable-implicit-cdi=true + +echo "Stopping Application AIDRTagger." +PID=$(ps -eo pid,cmd | grep [q]cri.aidr.predict.Controller | awk '{print $1}') +kill -9 $PID + +echo "Starting Application AIDRTagger." +cd $AIDR_HOME/aidr-tagger +nohup java -Xmx2048m -cp $GLASSFISH_HOME/glassfish/lib/gf-client.jar:target/aidr-tagger-1.0-jar-with-dependencies.jar:lib-non-maven/* qa.qcri.aidr.predict.Controller & fi if [ "$RUN_DDL" == "deploy_db" ] diff --git a/profiles/dev/config.properties b/profiles/dev/config.properties index f92e30fd2..871dcc04d 100644 --- a/profiles/dev/config.properties +++ b/profiles/dev/config.properties @@ -23,14 +23,14 @@ SQL_SERVER_TYPE=mysql HIBERNATE_DIALECT=org.hibernate.dialect.MySQLDialect #Server URL is used for persister download URL and twitter callback in aidr-manager -AIDR_SERVER_URL=localhost:8080 +AIDR_SERVER_URL= COLLECTOR_HOST=localhost -COLLECTOR_PORT=8080 +COLLECTOR_PORT=8084 COLLECTOR_WEB_APP_CONTEXT=AIDRCollector PERSISTER_HOST=localhost -PERSISTER_PORT=8080 +PERSISTER_PORT=8084 PERSISTER_WEB_APP_CONTEXT=AIDRPersister MANAGER_HOST=localhost @@ -38,20 +38,20 @@ MANAGER_PORT=8080 MANAGER_WEB_APP_CONTEXT=AIDRFetchManager DB_MANAGER_HOST=localhost -DB_MANAGER_PORT=8080 +DB_MANAGER_PORT=8084 DB_MANAGER_HBM2DDL=update DB_MANAGER_JNDI=JNDI/aidr_db_manager OUTPUT_API_HOST=localhost -OUTPUT_API_PORT=8080 +OUTPUT_API_PORT=8084 OUTPUT_API_WEB_APP_CONTEXT=AIDROutput TAGGER_API_HOST=localhost -TAGGER_API_PORT=8080 +TAGGER_API_PORT=8084 TAGGER_API_WEB_APP_CONTEXT=AIDRTaggerAPI TRAINER_API_HOST=localhost -TRAINER_API_PORT=8080 +TRAINER_API_PORT=8084 TRAINER_API_WEB_APP_CONTEXT=AIDRTrainerAPI MANAGER_DB_NAME=aidr_fetch_manager @@ -76,23 +76,23 @@ ANALYTICS_DB_PASSWORD=aidr_admin ANALYTICS_DB_HBM2DDL=update ANALYTICS_JNDI=JNDI/aidr_analysis -DEFAULT_PERSISTER_FILE_PATH=C:/Users/Latika/Desktop/Metacube/QCRI/temp/sc/aidr/data/persister/ +DEFAULT_PERSISTER_FILE_PATH=/sc/aidr/data/persister/ # Twitter specific configurations -twitter.consumerKey=hEIRfBfxNt8HlnfmRGkwrqIGb -twitter.consumerSecret=swVbpPSxJsSFP0tifvQ1vVxhMDY3bpfdZT8nHOOfC8EIKMzTuD +twitter.consumerKey= +twitter.consumerSecret= # root path of aidr-tagger. Path to the config.properties file is derived # through this. -TAGGER_HOME=C:/Users/Latika/Desktop/Metacube/QCRI/AIDR/dev_new/aidr-tagger +TAGGER_HOME=/home/aidr-tagger # Path to a directory where trained models are stored # this location is used by aidr-tagger.jar -MODEL_STORAGE_PATH=C:/Users/Latika/Desktop/Metacube/QCRI/temp/model +MODEL_STORAGE_PATH=model #Comma separated list of recipient emails #In case of fatal errors in the system, emails will be sent -RECIPIENT_EMAILS=latika.bhurani@metacube.com,kushalkant.goyal@metacube.com +RECIPIENT_EMAILS= ############################# # OPTIONAL configuration @@ -121,12 +121,13 @@ REMOTE_TASK_MANAGER_JNDI_PREFIX=java:global/AIDRTaskManager/aidr-task-manager-ej REMOTE_DB_MANAGER_JNDI_PREFIX=java:global/AIDRDBManager/aidr-db-manager-ejb-1.0 #Email configuration -SENDER_EMAIL=aidr.qcri@gmail.com -SENDER_PASS=qcrisc2015 +SENDER_EMAIL= +SENDER_PASS= SMTP_HOST=smtp.gmail.com SMTP_PORT=587 -MAIL_SUBJECT=Twitter Exception in collection: -MAIL_BODY= Unable to start the collection within the max retry attempts. Caused by: +MAIL_SUBJECT= +MAIL_BODY= + ############################# # FIXED configuration # Values should not be changed, as they may @@ -141,5 +142,5 @@ STATUS_CODE_WARNING=WARNING STATUS_CODE_FAILED=FAILED # Log settings to be used in all modules -LOG_LEVEL=DEBUG -LOG_LOCATION=C:/Users/Latika/Desktop/Metacube/QCRI/temp/var/log/aidr \ No newline at end of file +LOG_LEVEL=INFO +LOG_LOCATION=/var/log/aidr From 1045070cc8f2b3887c437730248ea91d2036375f Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Fri, 31 Jul 2015 12:24:32 +0530 Subject: [PATCH 23/44] Logging in AIDR-Output --- .../output/getdata/ChannelBufferManager.java | 11 ++++--- .../output/getdata/GetBufferedAIDRData.java | 5 +--- .../qcri/aidr/output/stream/AsyncStream.java | 3 -- .../aidr/output/utils/AIDROutputPing.java | 30 ++----------------- .../output/utils/JedisConnectionObject.java | 6 ---- .../aidr/output/utils/JsonDataFormatter.java | 12 +++----- 6 files changed, 12 insertions(+), 55 deletions(-) diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java index 987037014..394e6d64d 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/ChannelBufferManager.java @@ -134,7 +134,7 @@ public void initiateChannelBufferManager(final String channelRegEx) { logger.info("Created pattern subscription for pattern: " + channelRegEx); } catch (Exception e) { isSubscribed = false; - logger.error("Fatal exception occurred attempting subscription: " + e.toString()); + logger.error("Fatal exception occurred while attempting redis subscription: " + e.toString()); OutputErrorHandler.sendErrorMail(e.getLocalizedMessage(), "Fatal error! Couldn't establish connection to REDIS!"); } if (isSubscribed) { @@ -187,7 +187,7 @@ public void manageChannelBuffers(final String subscriptionPattern, final String createChannelQueue(channelName); addMessageToChannelBuffer(channelName, receivedMessage); logger.info("Created new channel: " + channelName); - System.out.println("[manageChannelBuffers] Created new channel: " + channelName); + //System.out.println("[manageChannelBuffers] Created new channel: " + channelName); } long currentTime = new Date().getTime(); @@ -207,7 +207,7 @@ public void manageChannelBuffers(final String subscriptionPattern, final String } statusFlags.clear(); } catch (Exception e) { - logger.error(e.getMessage()); + logger.error("Error while checking publiclyListedFlag for running collections "+e.getMessage()); } } lastPublicFlagCheckedTime = new Date().getTime(); @@ -458,7 +458,7 @@ public void run() { subscriberJedis.psubscribe(aidrSubscriber, channelRegEx); } catch (JedisConnectionException e) { logger.error("AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx, e); - System.out.println("[subscribeToChannel] AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx); + //System.out.println("[subscribeToChannel] AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx); stopSubscription(); Thread.currentThread().interrupt(); } /*finally { @@ -490,7 +490,7 @@ private void stopSubscription() { jedisConn.returnJedis(subscriberJedis); subscriberJedis = null; logger.info("Stopsubscription completed..."); - System.out.println("[stopSubscription] Stopsubscription completed..."); + //System.out.println("[stopSubscription] Stopsubscription completed..."); } } catch (Exception e) { logger.error("Failed to return Jedis resource"); @@ -563,7 +563,6 @@ public void close() { deleteAllChannelBuffers(); shutdownAndAwaitTermination(); logger.info("All done, fetch service has been shutdown..."); - System.out.println("[close] All done, fetch service has been shutdown..."); } /** diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/GetBufferedAIDRData.java b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/GetBufferedAIDRData.java index 42f59ae7f..721a7e74a 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/GetBufferedAIDRData.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/getdata/GetBufferedAIDRData.java @@ -182,7 +182,7 @@ public Response getLatestBufferedAIDRData(@QueryParam("callback") String callbac } //inRequests.decrementAndGet(); logger.error("Error in jedis connection. Bailing out..."); - System.err.println("[getLatestBufferedAIDRData] Error in jedis connection. Bailing out..."); + //System.err.println("[getLatestBufferedAIDRData] Error in jedis connection. Bailing out..."); return returnEmptyJson(callbackName); } @@ -511,12 +511,9 @@ public void contextInitialized(ServletContextEvent sce) { // Most important action - setup channel buffering thread if (null == cbManager) { logger.info("Initializing channel buffer manager with regEx pattern: " + CHANNEL_REG_EX); - System.out.println("[contextInitialized] Initializing channel buffer manager with regEx pattern: " + CHANNEL_REG_EX); - //cbManager = new ChannelBufferManager(CHANNEL_REG_EX); cbManager = new ChannelBufferManager(); cbManager.initiateChannelBufferManager(CHANNEL_REG_EX); logger.info("Done initializing channel buffer manager with regEx pattern: " + CHANNEL_REG_EX); - System.out.println("[contextInitialized] Done initializing channel buffer manager with regEx pattern: " + CHANNEL_REG_EX); } channelSelector = new SimpleFairScheduler(); logger.info("Context Initialized"); diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java b/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java index 977ca6356..7d93709ee 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/stream/AsyncStream.java @@ -84,9 +84,6 @@ public class AsyncStream implements ServletContextListener { @Override public void contextInitialized(ServletContextEvent sce) { -// AIDROutputConfig configuration = new AIDROutputConfig(); -// HashMap configParams = configuration.getConfigProperties(); - // Now initialize shared jedis connection object and thread pool object jedisConn = new JedisConnectionObject(redisHost, redisPort); executorService = Executors.newCachedThreadPool(); diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/AIDROutputPing.java b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/AIDROutputPing.java index dff3fd982..42b6440d3 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/AIDROutputPing.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/AIDROutputPing.java @@ -15,35 +15,21 @@ */ package qa.qcri.aidr.output.utils; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.HashMap; -import javax.ws.rs.Consumes; import javax.ws.rs.GET; -import javax.ws.rs.POST; import javax.ws.rs.Path; -import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.log4j.Logger; import qa.qcri.aidr.common.code.Configurator; -import qa.qcri.aidr.common.code.ResponseWrapperNEW; import redis.clients.jedis.Jedis; import redis.clients.jedis.exceptions.JedisConnectionException; -import twitter4j.ResponseList; -import twitter4j.Twitter; -import twitter4j.TwitterException; -import twitter4j.TwitterFactory; -import twitter4j.User; -import twitter4j.auth.AccessToken; @Path("/manage") @@ -51,7 +37,7 @@ public class AIDROutputPing { public static JedisConnectionObject jedisConn; // we need only a single instance of JedisConnectionObject running in background private static String host = "localhost"; - private static int port = 1978; + private static int port = 6379; private static HashMapAPIHashMap = null; @@ -64,21 +50,9 @@ public AIDROutputPing() { public AIDROutputPing(final String host, final int port) { - // AIDROutputConfig configuration = new AIDROutputConfig(); - // HashMap configParams = configuration.getConfigProperties(); Configurator configurator = OutputConfigurator.getInstance(); AIDROutputPing.host = configurator.getProperty(OutputConfigurationProperty.REDIS_HOST); AIDROutputPing.port = Integer.parseInt(configurator.getProperty(OutputConfigurationProperty.REDIS_PORT)); - /* - if (configParams.get("logger").equalsIgnoreCase("log4j")) { - // For now: set up a simple configuration that logs on the console - // PropertyConfigurator.configure("log4j.properties"); - // BasicConfigurator.configure(); // initialize log4j logging - } - if (configParams.get("logger").equalsIgnoreCase("slf4j")) { - System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "INFO"); // set logging level for slf4j - } - */ APIHashMap = new HashMap(); // Register available REST APIs diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JedisConnectionObject.java b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JedisConnectionObject.java index 09430c598..2850cd25d 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JedisConnectionObject.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JedisConnectionObject.java @@ -21,8 +21,6 @@ public class JedisConnectionObject { // Jedis related private static JedisPoolConfig poolConfig = null; // only one JedisPoolConfig per servlet instance private static JedisPool pool = null; // only one JedisPool per servlet instance - private Jedis subscriberJedis = null; // one for each GET request - private static OutputConfigurator configProperties = OutputConfigurator.getInstance(); public static String redisHost = configProperties.getProperty(OutputConfigurationProperty.REDIS_HOST); public static int redisPort = Integer.valueOf(configProperties.getProperty(OutputConfigurationProperty.REDIS_PORT)); @@ -38,10 +36,6 @@ public class JedisConnectionObject { * @param port port number to use for establishing connection */ public JedisConnectionObject(final String host, final int port) { - // For now: set up a simple configuration that logs on the console - //PropertyConfigurator.configure("log4j.properties"); // where to place the properties file? - //BasicConfigurator.configure(); // initialize log4j logging - //System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "INFO"); // set logging level for slf4j allotedJedis = new ConcurrentHashMap(); redisHost = host; diff --git a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JsonDataFormatter.java b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JsonDataFormatter.java index e3f51850e..2ecff8c18 100644 --- a/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JsonDataFormatter.java +++ b/aidr-output/src/main/java/qa/qcri/aidr/output/utils/JsonDataFormatter.java @@ -17,8 +17,6 @@ public class JsonDataFormatter { String callbackName = null; int count = 0; public JsonDataFormatter(String callbackName) { - //BasicConfigurator.configure(); // configuration for log4j logging - //System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "INFO"); // set logging level for slf4j this.callbackName = callbackName; count = 0; } @@ -33,16 +31,16 @@ private StringBuilder setPrefix(boolean isArray) { jsonDataList.append(callbackName).append("("); if (isArray) jsonDataList.append("["); - + return jsonDataList; } private StringBuilder setSuffix(StringBuilder jsonDataList, boolean isArray) { if (isArray) - jsonDataList.append("]"); + jsonDataList.append("]"); if (callbackName != null) - jsonDataList.append(")"); - + jsonDataList.append(")"); + return jsonDataList; } @@ -176,6 +174,4 @@ public StringBuilder createStreamingList(List bufferedMessages, int mess return jsonDataList; } - - } From 09824be61e1b3e5949d7372d9edf45d286947ece Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Fri, 31 Jul 2015 14:23:23 +0530 Subject: [PATCH 24/44] Commit for #98777594 : aidr-collector send email alert if not functioning properly --- .../collector/collectors/JedisPublisher.java | 2 + .../collectors/TwitterStatusListener.java | 27 +---------- .../init/CollectorStartStopController.java | 5 +- .../collector/utils/CollectorErrorLog.java | 46 +++++++++++++++++++ 4 files changed, 51 insertions(+), 29 deletions(-) create mode 100644 aidr-collector/src/main/java/qa/qcri/aidr/collector/utils/CollectorErrorLog.java diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/JedisPublisher.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/JedisPublisher.java index 847b019f5..e42c4e762 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/JedisPublisher.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/JedisPublisher.java @@ -10,6 +10,7 @@ import qa.qcri.aidr.collector.utils.CollectorConfigurator; import qa.qcri.aidr.collector.utils.CollectorConfigurationProperty; +import qa.qcri.aidr.collector.utils.CollectorErrorLog; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @@ -46,6 +47,7 @@ public static JedisPublisher newInstance() { return new JedisPublisher(jedis); } catch (JedisConnectionException e) { logger.severe("Could not establish Redis connection. Is Redis running?"); + CollectorErrorLog.sendErrorMail("Redis Connection", e.getMessage()); throw e; } } diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/TwitterStatusListener.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/TwitterStatusListener.java index c12cbb175..fe61a2e88 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/TwitterStatusListener.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/collectors/TwitterStatusListener.java @@ -24,6 +24,7 @@ import qa.qcri.aidr.collector.java7.Predicate; import qa.qcri.aidr.collector.utils.CollectorConfigurationProperty; import qa.qcri.aidr.collector.utils.CollectorConfigurator; +import qa.qcri.aidr.collector.utils.CollectorErrorLog; import qa.qcri.aidr.collector.utils.GenericCache; import twitter4j.ConnectionLifeCycleListener; import twitter4j.StallWarning; @@ -173,9 +174,7 @@ else if(((TwitterException) ex).getStatusCode() == 503) task.setStatusCode(configProperties.getProperty(CollectorConfigurationProperty.STATUS_CODE_COLLECTION_ERROR)); if(task.getStatusCode().equals(configProperties.getProperty(CollectorConfigurationProperty.STATUS_CODE_COLLECTION_ERROR))) - sendErrorMail(task.getCollectionCode(),ex.toString()); - //EmailClient.sendErrorMail(task.getCollectionCode(),ex.toString()); - + CollectorErrorLog.sendErrorMail(task.getCollectionCode(),ex.toString()); try { Thread.sleep(timeToSleep*1000); } catch (InterruptedException ignore) { @@ -222,26 +221,4 @@ public void onCleanUp() { // TODO Auto-generated method stub } - - public void sendErrorMail(String code, String errorMsg) { - Response clientResponse = null; - Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build(); - try { - WebTarget webResource = client.target(configProperties.getProperty(CollectorConfigurationProperty.TAGGER_REST_URI) - + "/misc/sendErrorEmail"); - - Form form = new Form(); - form.param("module", "AIDRCollector"); - form.param("code", code); - form.param("description", errorMsg); - - clientResponse = webResource.request().post( - Entity.entity(form,MediaType.APPLICATION_FORM_URLENCODED),Response.class); - if (clientResponse.getStatus() != 200) { - logger.warn("Couldn't contact AIDRTaggerAPI for sending error message"); - } - } catch (Exception e) { - logger.error("Error in contacting AIDRTaggerAPI: " + clientResponse); - } - } } \ No newline at end of file diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/init/CollectorStartStopController.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/init/CollectorStartStopController.java index 6aaa4e133..74363fbb4 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/init/CollectorStartStopController.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/init/CollectorStartStopController.java @@ -47,8 +47,6 @@ private void startup() { String startDate = dateFormat.format(cal.getTime()); GenericCache.getInstance().setCollectorStatus( new CollectorStatus(startDate, "RUNNING", 0)); - System.out.println("AIDR-Collector: Startup procedure completed @ " - + startDate); logger.info("AIDR-Collector: Startup procedure completed @ " + startDate); } @@ -68,8 +66,7 @@ private void shutdown() { try { Thread.sleep(1000); } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.warn("Error : " + e.getMessage()); } } diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/utils/CollectorErrorLog.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/utils/CollectorErrorLog.java new file mode 100644 index 000000000..cc2b9a906 --- /dev/null +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/utils/CollectorErrorLog.java @@ -0,0 +1,46 @@ +/** + * + */ +package qa.qcri.aidr.collector.utils; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Form; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.log4j.Logger; +import org.glassfish.jersey.jackson.JacksonFeature; + +/** + * @author Latika + * TODO centralize in all modules + */ +public class CollectorErrorLog { + + private static Logger logger = Logger.getLogger(CollectorErrorLog.class); + + public static void sendErrorMail(String code, String errorMsg) { + Response clientResponse = null; + Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build(); + try { + WebTarget webResource = client.target(CollectorConfigurator.getInstance() + .getProperty(CollectorConfigurationProperty.TAGGER_REST_URI) + "/misc/sendErrorEmail"); + + Form form = new Form(); + form.param("module", "AIDRCollector"); + form.param("code", code); + form.param("description", errorMsg); + + clientResponse = webResource.request().post( + Entity.entity(form,MediaType.APPLICATION_FORM_URLENCODED),Response.class); + if (clientResponse.getStatus() != 200) { + logger.warn("Couldn't contact AIDRTaggerAPI for sending error message"); + } + } catch (Exception e) { + logger.error("Error in contacting AIDRTaggerAPI: " + clientResponse); + } + } +} From c8e52a999b172c247270cb3602827c93c2e7dd18 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Fri, 31 Jul 2015 16:25:40 +0530 Subject: [PATCH 25/44] Added logs in AIDR-Persister. --- .../qa/qcri/aidr/io/FileSystemOperations.java | 23 +++++----- .../java/qa/qcri/aidr/io/ReadWriteCSV.java | 34 ++++---------- .../aidr/persister/api/Persist2FileAPI.java | 9 ++-- .../persister/api/Persister4CollectorAPI.java | 2 - .../collction/CollectionSubscriber.java | 10 +++-- .../collction/RedisCollectionPersister.java | 4 +- .../collector/CollectorSubscriber.java | 6 +-- .../collector/RedisCollectorPersister.java | 6 +-- .../tagger/RedisTaggerPersister.java | 5 ++- .../persister/tagger/TaggerSubscriber.java | 3 -- .../qcri/aidr/redis/JedisConnectionPool.java | 8 ++-- .../qa/qcri/aidr/utils/ClassifiedTweet.java | 2 - .../qa/qcri/aidr/utils/CompressBuffer.java | 28 ++++++------ .../qa/qcri/aidr/utils/DownloadJsonType.java | 7 ++- .../qa/qcri/aidr/utils/FileCompressor.java | 9 ++-- .../qa/qcri/aidr/utils/JsonDeserializer.java | 44 ++++++++----------- .../qa/qcri/aidr/utils/MD5HashGenerator.java | 7 ++- .../aidr/utils/PersisterErrorHandler.java | 43 ++++++++++++++++++ .../main/java/qa/qcri/aidr/utils/Tweet.java | 7 +++ 19 files changed, 137 insertions(+), 120 deletions(-) create mode 100644 aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterErrorHandler.java diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/io/FileSystemOperations.java b/aidr-persister/src/main/java/qa/qcri/aidr/io/FileSystemOperations.java index 9f2d7bfb7..5a9d8ff42 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/io/FileSystemOperations.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/io/FileSystemOperations.java @@ -8,6 +8,7 @@ import java.io.File; import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; @@ -21,7 +22,8 @@ public class FileSystemOperations { - + private static Logger logger = Logger.getLogger(FileSystemOperations.class); + public static ArrayList get100KFilesList(String collectionCode){ String filesPath = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/"; File folder = new File(filesPath); @@ -38,13 +40,11 @@ public static ArrayList get100KFilesList(String collectionCode){ if (currentVolN >= volNum) { volNum = currentVolN; fileNameList.add(currentFileName); - } } } } } - return null; } @@ -79,7 +79,6 @@ public static int getLatestFileVolumeNumber(String collectionCode) { File folder = new File(filesPath); File[] listOfFiles = folder.listFiles(); Integer volNum = 1; - String fileName = ""; if (!(listOfFiles == null)) { for (int i = 0; i < listOfFiles.length; i++) { if (listOfFiles[i].isFile()) { @@ -130,7 +129,7 @@ public static int getLatestFileVolumeNumber4Tagger(String collectionCode) { public static List getAllJSONFileVolumes(String collectionCode) { String filesPath = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/"; - List fileNames = new ArrayList(); + List fileNames = new ArrayList(); File folder = new File(filesPath); File[] listOfFiles = folder.listFiles(); for (int i = 0; i < listOfFiles.length; i++) { @@ -152,7 +151,7 @@ public static List getClassifiedFileVolumes(String collectionCode) { String filesPath = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/"; //String filesPath = getProperty("DEFAULT_PERSISTER_FILE_PATH") + collectionCode + "/"; - List fileNames = new ArrayList(); + List fileNames = new ArrayList(); File folder = new File(filesPath); File[] listOfFiles = folder.listFiles(); for (int i = 0; i < listOfFiles.length; i++) { @@ -178,14 +177,13 @@ public static boolean deleteFile(String fileName) { isDeleted = true; } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while delting the file: "+fileName +"\t" +e.getMessage()); } return isDeleted; } - public int countNumberofLines(String filename) throws IOException { - InputStream is = new BufferedInputStream(new FileInputStream(filename)); - try { + public int countNumberofLines(String filename) throws IOException{ + try(InputStream is = new BufferedInputStream(new FileInputStream(filename));) { byte[] c = new byte[1024]; int count = 0; int readChars = 0; @@ -199,8 +197,9 @@ public int countNumberofLines(String filename) throws IOException { } } return (count == 0 && !empty) ? 1 : count; - } finally { - is.close(); + } catch(IOException e){ + logger.error("IOException in file: "+filename); + throw e; } } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/io/ReadWriteCSV.java b/aidr-persister/src/main/java/qa/qcri/aidr/io/ReadWriteCSV.java index adaaa81e8..15b280c94 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/io/ReadWriteCSV.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/io/ReadWriteCSV.java @@ -12,7 +12,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; @@ -20,35 +19,29 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.log4j.Logger; +import org.glassfish.jersey.jackson.JacksonFeature; import org.supercsv.cellprocessor.Optional; import org.supercsv.cellprocessor.constraint.NotNull; import org.supercsv.cellprocessor.ift.CellProcessor; +import org.supercsv.encoder.DefaultCsvEncoder; +import org.supercsv.exception.SuperCsvCellProcessorException; import org.supercsv.io.CsvBeanWriter; import org.supercsv.io.CsvMapWriter; import org.supercsv.io.ICsvBeanWriter; import org.supercsv.io.ICsvMapWriter; import org.supercsv.prefs.CsvPreference; -import org.supercsv.encoder.DefaultCsvEncoder; -import org.supercsv.exception.SuperCsvCellProcessorException; -//import qa.qcri.aidr.common.logging.ErrorLog; -import qa.qcri.aidr.common.values.SystemProperties; +import qa.qcri.aidr.common.filter.NominalLabel; +import qa.qcri.aidr.utils.ClassifiedTweet; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; -import qa.qcri.aidr.common.filter.NominalLabel; import qa.qcri.aidr.utils.Tweet; -import org.apache.log4j.Logger; -import org.glassfish.jersey.jackson.JacksonFeature; - -import qa.qcri.aidr.utils.ClassifiedTweet; - public class ReadWriteCSV { - //private static final int BUFFER_SIZE = 10 * 1024 * 1024; private static Logger logger = Logger.getLogger(ReadWriteCSV.class); - //private static ErrorLog elog = new ErrorLog(); private String collectionCode = null; @@ -62,8 +55,6 @@ public class ReadWriteCSV { private static int countWritten = 0; - private String taggerMainUrl= "taggerMainUrl"; - public ReadWriteCSV(String collectionCode) { this.collectionCode = collectionCode; } @@ -172,9 +163,7 @@ public ICsvMapWriter getCSVMapWriter(String fileToWrite) { .useEncoder(new DefaultCsvEncoder()) .build() ); } catch (IOException e) { - // TODO Auto-generated catch block - logger.error("Error in creating CSV Bean writer!"); - logger.error("Exception",e); + logger.error("Error in creating CSV Bean writer!"+e); } return null; } @@ -207,16 +196,12 @@ public ICsvBeanWriter writeCollectorTweetIDSCSV(ICsvBeanWriter beanWriter, List< beanWriter.write(tweet, header, processors); } } catch (SuperCsvCellProcessorException e) { - //Logger.getLogger(ReadWriteCSV.class.getName()).log(Level.SEVERE, "[writeCollectorTweetIDSCSV] SuperCSV error"); logger.error(collectionDIR + ": SuperCSV error"); - //e.printStackTrace(); } } } catch (IOException ex) { - //Logger.getLogger(ReadWriteCSV.class.getName()).log(Level.SEVERE, "[writeCollectorTweetIDSCSV] IO Exception occured"); logger.error(collectionDIR + ": IO Exception occured"); - //ex.printStackTrace(); } //return fileName+".csv"; return beanWriter; @@ -259,8 +244,7 @@ public ICsvMapWriter writeClassifiedTweetIDsCSV(String[] runningHeader, ICsvMapW mapWriter.write(tweetToWrite, runningHeader, processors); ++countWritten; } catch (SuperCsvCellProcessorException e) { - //logger.error(collectionDIR + ": SuperCSV error. Offending tweet: " + tweet.getTweetID()); - //logger.error(elog.toStringException(e)); + logger.error(collectionDIR + ": SuperCSV error. Offending tweet: " + tweet.getTweetID()); } catch (IOException e) { logger.error(collectionDIR + "IOException in writing tweet: " + tweet.getTweetID()); } @@ -289,13 +273,11 @@ public ICsvBeanWriter writeCollectorTweetsCSV(List tweetsList, String col beanWriter.write(tweet, header, processors); } catch (SuperCsvCellProcessorException e) { logger.error(collectionDIR + ": SuperCSV error"); - //e.printStackTrace(); } } } catch (IOException ex) { logger.error(collectionDIR + ": IO Exception occured"); - //ex.printStackTrace(); } return beanWriter; } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persist2FileAPI.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persist2FileAPI.java index e5cd4c8be..c85b652be 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persist2FileAPI.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persist2FileAPI.java @@ -53,7 +53,6 @@ public Response generateCSVFromListFiltered(HumanLabeledDocumentListWrapper post try { DeserializeFilters des = new DeserializeFilters(); - System.out.println("constraints string received = " + postBody.getQueryString()); JsonQueryList queryList = des.deserializeConstraints(postBody.getQueryString()); JsonDeserializer jsonD = new JsonDeserializer(); @@ -81,7 +80,7 @@ public Response generateCSVFromListFiltered(HumanLabeledDocumentListWrapper post .header("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With") .build(); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while genrating filtered csv for collection: "+ collectionCode +"\t"+e.getStackTrace()); return Response.ok(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.STATUS_CODE_ERROR)).build(); } } @@ -178,7 +177,7 @@ public Response generateTweetsIDSCSVFromListFiltered(HumanLabeledDocumentListWra .build(); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while genrating filtered tweetIds csv for collection: "+ collectionCode +"\t"+e.getStackTrace()); return Response.ok( PersisterConfigurator.getInstance().getProperty( PersisterConfigurationProperty.STATUS_CODE_ERROR)) @@ -220,7 +219,7 @@ public Response generateJSONFromListFiltered(HumanLabeledDocumentListWrapper pos logger.info("Returning JSON object: " + ResultStatus.getUIWrapper(collectionCode, PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_CHANGE_NOTIFY_MSG), fileName, true)); return Response.ok(obj.toJSONString()).build(); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while genrating filtered json for collection: "+ collectionCode +"\t"+e.getStackTrace()); return Response.ok(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.STATUS_CODE_ERROR)).build(); } } @@ -311,7 +310,7 @@ public Response generateTweetsIDSJSONFromListFiltered(HumanLabeledDocumentListWr .build(); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while genrating filtered tweetIds Json for collection: "+ collectionCode +"\t"+e.getStackTrace()); return Response.ok(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.STATUS_CODE_ERROR)).build(); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectorAPI.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectorAPI.java index ba6518f63..9fdc9e3b4 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectorAPI.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/api/Persister4CollectorAPI.java @@ -71,7 +71,6 @@ public Response startPersister(@QueryParam("file") String fileLocation, @QueryPa return Response.ok(response).build(); } }catch (Exception ex) { - //Logger.getLogger(Persister4CollectorAPI.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + ": Failed to start persister"); } return Response.ok(response).build(); @@ -93,7 +92,6 @@ public Response stopPersister(@QueryParam("collectionCode") String collectionCod response = "Persistance of [" + collectionCode + "] has been stopped."; return Response.ok(response).build(); } catch (InterruptedException ex) { - //Logger.getLogger(Persister4CollectorAPI.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + ": Failed to stop persister"); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/CollectionSubscriber.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/CollectionSubscriber.java index 1c1d3c9ab..08133a78b 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/CollectionSubscriber.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/CollectionSubscriber.java @@ -95,7 +95,7 @@ private void createNewFile() { file.createNewFile(); } } catch (IOException ex) { - logger.error(collectionCode + " error in creating new file at location " + collectionDir); + logger.error(collectionCode + " Error in creating new file at location " + collectionDir); } } @@ -108,6 +108,8 @@ private String createNewDirectory() { if (result) { logger.info("DIR created for collection: " + collectionCode); return persisterDir + collectionCode + "/"; + }else{ + logger.error(collectionCode+ " Unable to create a new directory: "); } } @@ -119,7 +121,7 @@ private void createBufferWriter() { //out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(this.file, true), Charset.forName("UTF-8")), Integer.parseInt(getProperty("DEFAULT_FILE_WRITER_BUFFER_SIZE"))); out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(this.file, true)), Integer.parseInt(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_FILE_WRITER_BUFFER_SIZE))); } catch (IOException ex) { - logger.error(collectionCode + "Error in creating Buffered writer"); + logger.error(collectionCode + " Error in creating Buffered writer"); } } @@ -130,7 +132,7 @@ private void writeToFile(String message) { itemsWrittenToFile++; isTimeToCreateNewFile(); } catch (IOException ex) { - logger.error(collectionCode + "Error in writing to file"); + logger.error(collectionCode + " Error in writing to file"); } } @@ -151,7 +153,7 @@ public void closeFileWriting() { out.close(); } } catch (IOException ex) { - logger.error(collectionCode + "Error in closing file writer"); + logger.error(collectionCode + " Error in closing file writer"); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/RedisCollectionPersister.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/RedisCollectionPersister.java index 1f7ad8cf2..a72361448 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/RedisCollectionPersister.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collction/RedisCollectionPersister.java @@ -4,6 +4,7 @@ import org.apache.log4j.Logger; import qa.qcri.aidr.redis.JedisConnectionPool; +import qa.qcri.aidr.utils.PersisterErrorHandler; import redis.clients.jedis.Jedis; /** @@ -34,9 +35,8 @@ public RedisCollectionPersister(String fileName, String channel, String collecti subscriberJedis = connObject.getJedisConnection(); subscriber = new CollectionSubscriber(fileName, channel, collectionCode); } catch (Exception e) { - // TODO Auto-generated catch block logger.error(collectionCode + ": Error in subscribing to Redis"); - + PersisterErrorHandler.sendErrorMail(e.getLocalizedMessage(), "Error in subscribing to Redis for collection: "+collectionCode); connObject.close(subscriberJedis); subscriberJedis = null; subscriber = null; diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/CollectorSubscriber.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/CollectorSubscriber.java index 235aedf99..00cada38a 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/CollectorSubscriber.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/CollectorSubscriber.java @@ -96,7 +96,7 @@ private void createNewFile() { file.createNewFile(); } } catch (IOException ex) { - logger.error(collectionCode + " error in creating new file at location " + collectionDir); + logger.error(collectionCode + " Error in creating new file at location " + collectionDir); } } @@ -117,10 +117,8 @@ private String createNewDirectory() { private void createBufferWriter() { try { - //out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(this.file, true), Charset.forName("UTF-8")), Integer.parseInt(getProperty("DEFAULT_FILE_WRITER_BUFFER_SIZE"))); out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(this.file, true)), Integer.parseInt(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_FILE_WRITER_BUFFER_SIZE))); } catch (IOException ex) { - //Logger.getLogger(CollectorSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in creating Buffered writer"); } @@ -132,7 +130,6 @@ private void writeToFile(String message) { itemsWrittenToFile++; isTimeToCreateNewFile(); } catch (IOException ex) { - //Logger.getLogger(CollectorSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in writing to file"); } } @@ -152,7 +149,6 @@ public void closeFileWriting() { out.flush(); out.close(); } catch (IOException ex) { - //Logger.getLogger(CollectorSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in closing file writer"); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/RedisCollectorPersister.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/RedisCollectorPersister.java index d13c2b0de..8166aa978 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/RedisCollectorPersister.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/collector/RedisCollectorPersister.java @@ -10,6 +10,7 @@ import qa.qcri.aidr.redis.JedisConnectionPool; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; +import qa.qcri.aidr.utils.PersisterErrorHandler; import redis.clients.jedis.Jedis; @@ -40,9 +41,8 @@ public RedisCollectorPersister(String fileName, String collectionCode) throws In subscriberJedis = connObject.getJedisConnection(); subscriber = new CollectorSubscriber(fileName, collectionCode); } catch (Exception e) { - // TODO Auto-generated catch block logger.error(collectionCode + ": Error in subscribing to Redis"); - + PersisterErrorHandler.sendErrorMail(e.getLocalizedMessage(), "Error in subscribing to Redis for collection: "+collectionCode); connObject.close(subscriberJedis); subscriberJedis = null; subscriber = null; @@ -72,7 +72,6 @@ public void run() { connObject.close(subscriberJedis); // return jedis resource to JedisPool Thread.sleep(200); } catch (InterruptedException ex) { - //Logger.getLogger(RedisCollectorPersister.class.getName()).log(Level.SEVERE, null, ex); logger.warn(collectionCode + " error in closing Redis connection"); } } @@ -105,7 +104,6 @@ public void suspendMe() { try { t.join(); } catch (InterruptedException e) { - // TODO Auto-generated catch block logger.warn(collectionCode + ": Collector Persister Thread join interrupted"); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/RedisTaggerPersister.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/RedisTaggerPersister.java index e128530c2..fd66c692b 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/RedisTaggerPersister.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/RedisTaggerPersister.java @@ -10,6 +10,7 @@ import qa.qcri.aidr.redis.JedisConnectionPool; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; +import qa.qcri.aidr.utils.PersisterErrorHandler; import redis.clients.jedis.Jedis; @@ -39,7 +40,8 @@ public RedisTaggerPersister(String fileName, String collectionCode) throws Inter subscriberJedis = connObject.getJedisConnection(); subscriber = new TaggerSubscriber(fileName, collectionCode); } catch (Exception e) { - logger.error(collectionCode + " error in subscribing to Redis"); + logger.error(collectionCode + " Error in subscribing to Redis"); + PersisterErrorHandler.sendErrorMail(e.getLocalizedMessage(), "Error in subscribing to Redis for collection: "+collectionCode); connObject.close(subscriberJedis); subscriberJedis = null; subscriber = null; @@ -97,7 +99,6 @@ public void suspendMe() { try { t.join(); } catch (InterruptedException e) { - // TODO Auto-generated catch block logger.warn(collectionCode + ": Tagger Persister Thread join interrupted"); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/TaggerSubscriber.java b/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/TaggerSubscriber.java index 26e99d977..cc20ecbb5 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/TaggerSubscriber.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/persister/tagger/TaggerSubscriber.java @@ -113,7 +113,6 @@ private void createNewFile() { file.createNewFile(); } } catch (IOException ex) { - //Logger.getLogger(TaggerSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + " error in creating new file at location " + collectionDir); } } @@ -139,7 +138,6 @@ private void createBufferWriter() { out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(this.file, true)), Integer.parseInt(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_FILE_WRITER_BUFFER_SIZE))); } catch (IOException ex) { - //Logger.getLogger(TaggerSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in creating Buffered writer"); } @@ -182,7 +180,6 @@ public void closeFileWriting() { out.flush(); out.close(); } catch (IOException ex) { - //Logger.getLogger(TaggerSubscriber.class.getName()).log(Level.SEVERE, null, ex); logger.error(collectionCode + "Error in closing file writer"); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/redis/JedisConnectionPool.java b/aidr-persister/src/main/java/qa/qcri/aidr/redis/JedisConnectionPool.java index 656e3d65c..8a738aa27 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/redis/JedisConnectionPool.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/redis/JedisConnectionPool.java @@ -7,8 +7,10 @@ import org.apache.log4j.Logger; +import qa.qcri.aidr.common.util.EmailClient; import qa.qcri.aidr.utils.PersisterConfigurationProperty; import qa.qcri.aidr.utils.PersisterConfigurator; +import qa.qcri.aidr.utils.PersisterErrorHandler; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @@ -29,9 +31,6 @@ public synchronized Jedis getJedisConnection() { // koushik: removed static try { if (jedisPool == null) { JedisPoolConfig config = new JedisPoolConfig(); - //config.maxActive = 1000; - //config.maxIdle = 10; - //config.minIdle = 1; config.setMaxTotal(1000); config.setMaxIdle(10); config.setMinIdle(1); @@ -43,13 +42,12 @@ public synchronized Jedis getJedisConnection() { // koushik: removed static return jedisPool.getResource(); } catch (JedisConnectionException e) { logger.error("Could not establish Redis connection. Is the Redis running?"); + PersisterErrorHandler.sendErrorMail(e.getLocalizedMessage(), "Could not establish Redis connection. Is the Redis running?"); throw e; } } public synchronized void close(Jedis resource) { // koushik: removed static, added code to increase robustness - //jedisPool.returnResource(resource); - if (jedisPool != null) { try { if (resource != null) { diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java index 1a5d0bd7b..a96473d94 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/ClassifiedTweet.java @@ -307,7 +307,6 @@ public Date getDate(String timeString) { } public String setDateString(String timeString) { - //System.out.println("[setDateString] Received time string: " + timeString); DateFormat dateFormatISO = new SimpleDateFormat(DateFormatConfig.ISODateFormat); dateFormatISO.setTimeZone(TimeZone.getTimeZone("GMT")); @@ -316,7 +315,6 @@ public String setDateString(String timeString) { SimpleDateFormat formatter = new SimpleDateFormat(DateFormatConfig.StandardDateFormat); Date newDate = formatter.parse(timeString); if (newDate != null) setTimestamp(newDate.getTime()); - //System.out.println("[setDateString] Converted date: " + newDate.toString()); return dateFormatISO.format(newDate); } catch (ParseException e) { logger.error("Error in setting createdAt field = " + timeString); diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/CompressBuffer.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/CompressBuffer.java index a4844f85d..3574c1f06 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/CompressBuffer.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/CompressBuffer.java @@ -3,18 +3,18 @@ */ package qa.qcri.aidr.utils; -import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; -import java.io.FileReader; - import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; -public class CompressBuffer extends FileCompressor { +import org.apache.log4j.Logger; +public class CompressBuffer extends FileCompressor { + + private static Logger logger = Logger.getLogger(CompressBuffer.class.getName()); //ByteArrayOutputStream out = null; FileOutputStream out = null; ZipOutputStream outZip = null; @@ -44,7 +44,7 @@ public CompressBuffer(String folderLocation, String zippedFile) { ze = new ZipEntry(zippedFile); outZip.putNextEntry(ze); } catch (Exception e) { - e.printStackTrace(); + logger.error("IOException while compressing the buffer"+e); } } @@ -67,7 +67,7 @@ public void init(String folderLocation, String zippedFile) { ze = new ZipEntry(zippedFile); outZip.putNextEntry(ze); } catch (Exception e) { - e.printStackTrace(); + logger.error("IOException while initializing the compress buffer"+e); } } @@ -76,7 +76,7 @@ public void close() { outZip.closeEntry(); outZip.close(); } catch (Exception e) { - e.printStackTrace(); + logger.error("IOException while closing the compressed file"); } } @@ -89,7 +89,7 @@ public void zip(final byte[] data) { try { outZip.write(data, 0, data.length); } catch (Exception e) { - e.printStackTrace(); + logger.error("IOException while compressing the file "); } } @@ -103,7 +103,7 @@ public void zip(final String str) { byte[] buffer = str.trim().getBytes(); zip(buffer); } catch (Exception e) { - e.printStackTrace(); + logger.error("IOException while compressing the file "); } } @@ -150,7 +150,7 @@ public byte[] unzip(String zipFile, String unzippedFile) { return returnBuf; } catch(Exception e) { - e.printStackTrace(); + logger.error("IOException while unzipping the directory "); return null; } } @@ -180,7 +180,7 @@ public String unzipAsString(String zipFile, String unzippedFile) { return null; } } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception in unzipping the directory"+e); return null; } } @@ -195,12 +195,12 @@ public String unzipAsString(String zipFile) { String unzippedString = unzipAsString(zipFile, null); return unzippedString; } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception in unzipping the directory"+e); return null; } } - public static void main(String[] args) throws Exception { + /*public static void main(String[] args) throws Exception { CompressBuffer zipper = new CompressBuffer(); String collectionCode = "20150104-0348-SinhaKoushik-abcdertgh"; @@ -234,5 +234,5 @@ public static void main(String[] args) throws Exception { String unzippedString = zipper.unzipAsString(folderLocation + File.separator + "testZip.zip"); System.out.println("Done creating UNZIP file! unzipped string size = " + unzippedString.length()); System.out.println((unzippedBuffer != null ? unzippedString : "Error in unzip!")); - } + }*/ } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/DownloadJsonType.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/DownloadJsonType.java index 08dc1817f..b0b628fd9 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/DownloadJsonType.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/DownloadJsonType.java @@ -1,6 +1,5 @@ package qa.qcri.aidr.utils; -import org.apache.log4j.Logger; public enum DownloadJsonType { @@ -10,7 +9,7 @@ public enum DownloadJsonType { private final String value; private final String suffix; - private static Logger logger = Logger.getLogger(DownloadJsonType.class.getName()); + //private static Logger logger = Logger.getLogger(DownloadJsonType.class.getName()); public String getSuffix() { return this.suffix; @@ -69,9 +68,9 @@ public static String defaultSuffix() { return DownloadJsonType.TEXT_JSON.suffix; } - public static void main(String args[]) throws Exception { + /*public static void main(String args[]) throws Exception { DownloadJsonType d = DownloadJsonType.TEXT_JSON; System.out.println("suffix: " + DownloadJsonType.getSuffixString(d)); System.out.println("testing: " + DownloadJsonType.getDownloadJsonTypeFromString("TEXT_JSON")); - } + }*/ } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java index e341a3976..c72fe8279 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java @@ -16,9 +16,12 @@ import java.util.zip.ZipOutputStream; import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; public class FileCompressor { + private static Logger logger = Logger.getLogger(FileCompressor.class.getName()); + public static int BUFFER_SIZE = 1 << 16; private String fileName; @@ -75,7 +78,7 @@ public String zip() { return this.getOutFile(); } catch (Exception e) { - e.printStackTrace(); + logger.error("IOException while compressing the file: "+fileName); return null; } } @@ -106,10 +109,10 @@ public String unzip() { zis.closeEntry(); zis.close(); - System.out.println("Done unzipping file: " + this.getInputFileName() + ", created file = " + this.getUnzippedOutputFileName()); + //System.out.println("Done unzipping file: " + this.getInputFileName() + ", created file = " + this.getUnzippedOutputFileName()); return this.getOutUnzippedFile(); } catch(Exception ex) { - ex.printStackTrace(); + logger.error("IOException while unzipping the file"+ex); return null; } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java index d776767d1..ed1ec16e4 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java @@ -168,7 +168,6 @@ public Map generateClassifiedJson2TweetIdsCSV(String collectionC String fileToDelete = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/" + "Classified_" + collectionCode + "_tweetIds.csv"; logger.info(collectionCode + ": Deleteing file : " + fileToDelete); FileSystemOperations.deleteFile(fileToDelete); // delete if there exist a csv file with same name - //logger.info(fileNames); long lastCount = 0; long currentCount = 0; @@ -292,7 +291,6 @@ public Map generateClassifiedJson2TweetIdsCSVFiltered(final Stri if (queryList != null) tweetFilter.queryList.setConstraints(queryList); tweetFilter.buildMatcherArray(); - String lastLine = null; for (String file : fileNames) { if (downloadLimited && totalCount >= Integer.parseInt(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_TWEETID_VOLUME_LIMIT))) { break; @@ -305,7 +303,6 @@ public Map generateClassifiedJson2TweetIdsCSVFiltered(final Stri br = new ReversedLinesFileReader(f); String line; while ((line = br.readLine()) != null) { - lastLine = line; if (downloadLimited && totalCount >= Integer.parseInt(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_TWEETID_VOLUME_LIMIT))) { tweetsList.clear(); break; @@ -449,7 +446,7 @@ public int compare(File f1, File f2) { } } } catch (Exception ex) { - //Logger.getLogger(JsonDeserializer.class.getName()).log(Level.SEVERE, "JSON file parsing exception at line {0}", lineNumber); + logger.error("Error while parsing the json"+ex); } } if (!tweetsList.isEmpty()) { @@ -489,7 +486,6 @@ public String taggerGenerateJSON2CSV_100K_BasedOnTweetCount(String collectionCod ICsvMapWriter writer = null; try { - String folderLocation = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode; String fileNameforCSVGen = "Classified_" + collectionCode + "_last_100k_tweets"; fileName = fileNameforCSVGen + ".csv"; @@ -911,8 +907,7 @@ private Tweet getTweet(String line) { } } } catch (Exception ex) { - //Logger.getLogger(JsonDeserializer.class.getName()).log(Level.SEVERE, null, ex); - //System.out.println("[getTweet] Offending tweet: " + line); + logger.error("Exception while parsing the json to tweet"+ex); return null; } return tweet; @@ -922,9 +917,7 @@ public ClassifiedTweet getClassifiedTweet(String line) { return getClassifiedTweet(line, null); } - // getClassifiedTweet method using Gson library for parsing JSON string public ClassifiedTweet getClassifiedTweet(String line, String collectionCode) { - //System.out.println("Tweet to PARSE: " + line); ClassifiedTweet tweet = new ClassifiedTweet(); try { @@ -1037,7 +1030,7 @@ public ClassifiedTweet getClassifiedTweet(String line, String collectionCode) { } return tweet; } catch (Exception ex) { - //logger.error("exception", ex); + logger.error("Exception while parsing the json to classiffied tweet for the collection: "+collectionCode +"\t"+ ex); return null; } } @@ -1110,7 +1103,7 @@ public int compare(File f1, File f2) { break; } } catch (Exception ex) { - //Logger.getLogger(JsonDeserializer.class.getName()).log(Level.SEVERE, "JSON file parsing exception at line {0}", lineNumber); + logger.error("JSON file parsing exception"+ex); } } // end while br.close(); @@ -1206,7 +1199,7 @@ public int compare(File f1, File f2) { ++totalCount; } } catch (Exception ex) { - //Logger.getLogger(JsonDeserializer.class.getName()).log(Level.SEVERE, "JSON file parsing exception at line {0}", lineNumber); + logger.error("JSON file parsing exception"+ex); } } // end while br.close(); @@ -1321,7 +1314,7 @@ public int compare(File f1, File f2) { break; } } catch (Exception ex) { - //Logger.getLogger(JsonDeserializer.class.getName()).log(Level.SEVERE, "JSON file parsing exception at line {0}", lineNumber); + logger.error("JSON file parsing exception"+ex); } } // end while br.close(); @@ -1478,7 +1471,7 @@ public String taggerGenerateJSON2JSON_100K_BasedOnTweetCountFiltered( File folder = new File(folderLocation); File[] listOfFiles = folder.listFiles(); // to get only Tagger's files - ArrayList taggerFilesList = new ArrayList(); + ArrayList taggerFilesList = new ArrayList(); for (int i = 0; i < listOfFiles.length; i++) { if (StringUtils.startsWith(listOfFiles[i].getName(), (collectionCode + "_")) && StringUtils.containsIgnoreCase(listOfFiles[i].getName(), "vol")) { @@ -1495,7 +1488,6 @@ public int compare(File f1, File f2) { } }); - StringBuffer outputFile = new StringBuffer().append(folderLocation).append(fileName); beanWriter = new BufferedWriter(new FileWriter(outputFile.toString()), BUFFER_SIZE); if (DownloadJsonType.JSON_OBJECT.equals(jsonType)) { @@ -1548,7 +1540,7 @@ public int compare(File f1, File f2) { break; } } catch (Exception ex) { - //Logger.getLogger(JsonDeserializer.class.getName()).log(Level.SEVERE, "JSON file parsing exception at line {0}", lineNumber); + logger.error("JSON file parsing exception" + ex); } } // end while br.close(); @@ -1613,7 +1605,6 @@ public Map generateClassifiedJson2TweetIdsJSONFiltered( int totalCount = 0; try { - //List fileNames = FileSystemOperations.getClassifiedFileVolumes(collectionCode); List fileNames = FileSystemOperations.getAllJSONFileVolumes(collectionCode); Collections.sort(fileNames); Collections.reverse(fileNames); @@ -1758,7 +1749,7 @@ public String generateClassifiedList2CSV_100K_BasedOnTweetCountFiltered(String c fileName = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_DOWNLOAD_URL) + "/" + collectionCode + "/" + fileNameforCSVGen; } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while generating MD5Hash for user: "+userName); return null; } @@ -1822,14 +1813,14 @@ public String generateClassifiedList2CSV_100K_BasedOnTweetCountFiltered(String c tweetsList.clear(); } } catch (Exception e) { - logger.error("Exception", e); + logger.error("Exception in generateClassifiedList2CSV_100K_BasedOnTweetCountFiltered", e); return null; } finally { if (writer != null) { try { writer.close(); } catch (IOException e) { - logger.error("Exception", e); + logger.error("Exception in generateClassifiedList2CSV_100K_BasedOnTweetCountFiltered", e); return null; } } @@ -1855,7 +1846,7 @@ public Map generateClassifiedList2TweetIdsCSVFiltered(String col fileName = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_DOWNLOAD_URL) + "/" + collectionCode + "/" + fileNameforCSVGen; } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while generating MD5Hash for user: "+userName); return null; } String folderLocation = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode; @@ -1905,8 +1896,6 @@ public Map generateClassifiedList2TweetIdsCSVFiltered(String col tweetsList.add(tweet); } } - - } // end for int countToWrite = tweetsList.size(); if (downloadLimited) { @@ -1920,7 +1909,10 @@ public Map generateClassifiedList2TweetIdsCSVFiltered(String col totalCount += countToWrite; tweetsList.clear(); } - } finally { + } catch(Exception e){ + logger.error("Exception in generateClassifiedList2TweetIdsCSVFiltered"); + } + finally { if (writer != null) { try { writer.close(); @@ -1956,7 +1948,7 @@ public String generateClassifiedList2JSON_100K_BasedOnTweetCountFiltered(String fileName = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_DOWNLOAD_URL) + "/" + collectionCode + "/" + fileNameforGen; } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while generating MD5Hash for user: "+userName); return null; } @@ -2060,7 +2052,7 @@ public Map generateClassifiedList2TweetIdsJSONFiltered(String co fileName = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_DOWNLOAD_URL) + "/" + collectionCode + "/" + fileNameforGen; } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating MD5Hash for the user: "+userName); return null; } String fileToDelete = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/" + fileNameforGen; diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/MD5HashGenerator.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/MD5HashGenerator.java index 5e84e5bb4..7b36d64be 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/MD5HashGenerator.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/MD5HashGenerator.java @@ -6,8 +6,12 @@ import java.security.MessageDigest; -public class MD5HashGenerator { +import org.apache.log4j.Logger; +public class MD5HashGenerator { + + private static Logger logger = Logger.getLogger(MD5HashGenerator.class); + public MD5HashGenerator() {} public String getMD5Hash(String data) throws Exception { @@ -35,6 +39,7 @@ public String getMD5Hash(String data) throws Exception { //System.out.println("Digest(in hex format):: " + hexString.toString()); return hexString.toString(); } else { + logger.warn("MD5Hash won't be generated for null data"); throw new Exception("Data can't be null"); } } diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterErrorHandler.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterErrorHandler.java new file mode 100644 index 000000000..063c7045b --- /dev/null +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/PersisterErrorHandler.java @@ -0,0 +1,43 @@ +package qa.qcri.aidr.utils; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Form; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.log4j.Logger; +import org.glassfish.jersey.jackson.JacksonFeature; + +/** + * @author Kushal + * + */ +public class PersisterErrorHandler { + + private static Logger logger = Logger.getLogger(PersisterErrorHandler.class); + + public static void sendErrorMail(String code, String errorMsg) { + Response clientResponse = null; + Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build(); + try { + WebTarget webResource = client.target(PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.TAGGER_MAIN_URL) + + "/misc/sendErrorEmail"); + + Form form = new Form(); + form.param("module", "AIDRPersister"); + form.param("code", code); + form.param("description", errorMsg); + + clientResponse = webResource.request().post( + Entity.entity(form,MediaType.APPLICATION_FORM_URLENCODED),Response.class); + if (clientResponse.getStatus() != 200) { + logger.warn("Couldn't contact AIDRTaggerAPI for sending error message"); + } + } catch (Exception e) { + logger.error("Error in contacting AIDRTaggerAPI: " + clientResponse); + } + } +} diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/Tweet.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/Tweet.java index 26ce03193..c0776055d 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/Tweet.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/Tweet.java @@ -10,6 +10,10 @@ import java.io.Serializable; +import org.apache.log4j.Logger; + +import qa.qcri.aidr.io.ReadWriteCSV; + import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -17,6 +21,8 @@ @SuppressWarnings("serial") public class Tweet implements Document, Serializable{ + private static Logger logger = Logger.getLogger(Tweet.class); + private String tweetID; private String message; private String crisisName; @@ -179,6 +185,7 @@ public Tweet toTweet(String str) { } return tweet; } catch (Exception ex) { + logger.error("Unable to deserialize the json string to tweet"+ex); return null; } } From 25dcdaa84faedea70a54e5196616a8f5ae3b994e Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Fri, 31 Jul 2015 16:29:31 +0530 Subject: [PATCH 26/44] Commit for #97809356 : logging for aidr-analytics module --- .../analysis/api/GetTagDataStatistics.java | 3 - .../analysis/api/WriteStatisticsData.java | 9 +-- .../TagDataStatisticsResourceFacadeImp.java | 75 ++++++++++--------- .../service/GetTagDataStatisticsService.java | 4 - .../aidr/analysis/stat/ConfDataMapRecord.java | 2 - .../utils/AnalyticsConfigurationProperty.java | 2 +- .../analysis/utils/AnalyticsConfigurator.java | 3 +- .../analysis/utils/AnalyticsErrorLog.java | 46 ++++++++++++ .../analysis/utils/ChannelBufferManager.java | 53 ++++--------- .../src/main/resources/config.properties | 6 ++ .../src/main/resources/granularity.properties | 11 --- .../src/main/resources/log4j.properties | 2 +- 12 files changed, 110 insertions(+), 106 deletions(-) create mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsErrorLog.java delete mode 100644 aidr-analytics/src/main/resources/granularity.properties diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java index 781499d32..3752d558d 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/GetTagDataStatistics.java @@ -54,7 +54,6 @@ public Response getTagCountSumForAllAttributesFromTime(@PathParam("crisisCode") @PathParam("granularity") Long granularity, @DefaultValue("0") @QueryParam("startTime") Long startTime) { - System.out.println("local EJB in REST API = " + tagDataService); JSONObject json = tagDataService.getTagCountSumForAllAttributesFromTime(crisisCode, granularity, startTime); return Response.ok(json.toJSONString()).build(); } @@ -72,7 +71,6 @@ public Response getTagCountSumForAllAttributesFromTime(@PathParam("crisisCode") public Response getTagCountSumForAllGranularitiesFromTime(@PathParam("crisisCode") String crisisCode, @DefaultValue("0") @QueryParam("startTime") Long startTime) { - System.out.println("local EJB in REST API = " + tagDataService); JSONObject json = tagDataService.getTagCountSumByGranularity(crisisCode, startTime); return Response.ok(json.toJSONString()).build(); } @@ -94,7 +92,6 @@ public Response getTagCountSumFromTime(@PathParam("crisisCode") String crisisCod @PathParam("granularity") Long granularity, @DefaultValue("0") @QueryParam("startTime") Long startTime) { - System.out.println("local EJB in REST API = " + tagDataService); JSONObject json = tagDataService.getTagCountSumFromTime(crisisCode, attributeCode, granularity, startTime); return Response.ok(json.toJSONString()).build(); } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java index 9aebeedcf..f8b3fc7a3 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -82,7 +82,6 @@ public void contextInitialized(ServletContextEvent sce) { cbManager = new ChannelBufferManager(); cbManager.initiateChannelBufferManager(GetStatistics.CHANNEL_REG_EX); logger.info("Done initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); - System.out.println("[contextInitialized] Done initializing channel buffer manager with regEx pattern: " + GetStatistics.CHANNEL_REG_EX); } //tagDataMap = ChannelBufferManager.getTagDataMap(); //confDataMap = ChannelBufferManager.getConfDataMap(); @@ -94,11 +93,8 @@ public void contextInitialized(ServletContextEvent sce) { executorServicePool = cbManager.getExecutorServicePool(); if (executorServicePool != null) { executorServicePool.submit(t); - System.out.println("Executor thread pool initialized, submitted thread: " + t.getName() + " to pool = " + executorServicePool); + logger.info("Executor thread pool initialized, submitted thread: " + t.getName() + " to pool = " + executorServicePool); } - logger.info("Done initializing channel buffer manager"); - System.out.println("Done initializing channel buffer manager"); - System.out.print("Initialized with granularities: "); for (Long g: granularityList) { System.out.print(g + "\t"); } @@ -139,7 +135,7 @@ private ReturnCode writeOutputDataToTagDataDB(Long granularity,Long timestamp) { TagDataMapRecord tCount = (TagDataMapRecord) ChannelBufferManager.getTagDataMap().get(key); TagData t = new TagData(key.getCrisisCode(), timestamp, granularity, key.getAttributeCode(), key.getLabelCode(), tCount.getCount(granularity)); if (tCount.getCount(granularity) > 0) { - System.out.println("Will attempt persistence of tag key: " + key.toString()); + logger.error("Will attempt persistence of tag key: " + key.toString()); t.setMaxCreatedAt(tCount.getLastUpdateTime()); t.setMinCreatedAt(tCount.getFirstEntryTime()); tagDataEJB.writeData(t); @@ -216,6 +212,7 @@ public void run() { try { Thread.sleep(granularityList.get(0) - 1000); // sleep for the minimum granularity period } catch (InterruptedException e) { + logger.warn("Sleep thread interrupted."); } } } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java index 3e964e63b..a5a17365f 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java @@ -38,6 +38,7 @@ public EntityManager getEntityManager() { try { return em; } catch (Exception e) { + logger.error("getEntityManager failed"); throw new HibernateException("getEntityManager failed"); } } @@ -53,8 +54,7 @@ public int setEntityManager(EntityManager em) { logger.info("Skipping setter, since EntityManager already initialized to :" + this.em); return 0; } catch (Exception e) { - logger.error("EntityManager setting exception : " + em); - logger.error("exception", e); + logger.error("EntityManager setting exception : " + em, e); throw new HibernateException("setEntityManager failed"); } } @@ -65,7 +65,6 @@ public Session getCurrentSession() { return em.unwrap(Session.class); } catch (Exception e) { logger.error("exception: ", e); - e.printStackTrace(); throw new HibernateException("getCurrentSession failed"); } } @@ -75,13 +74,12 @@ public Session getCurrentSession() { public ReturnCode writeData(TagData tagData) { try { em.persist(tagData); - System.out.println("Success in persisting tag data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + logger.info("Success in persisting tag data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + ", " + tagData.getLabelCode() + ", " + tagData.getTimestamp() + ", " + tagData.getGranularity() + ": " + tagData.getCount()); return ReturnCode.SUCCESS; } catch (Exception e) { - System.out.println("Failure in persisting tag data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + logger.error("Failure in persisting tag data for: " + tagData.getCrisisCode() + ", " + tagData.getAttributeCode() + ", " + tagData.getLabelCode() + ", " + tagData.getTimestamp() + ", " + tagData.getGranularity() + ": " + tagData.getCount()); - e.printStackTrace(); logger.error("exception: ", e); return ReturnCode.ERROR; } @@ -102,8 +100,9 @@ public TagData getSingleDataByPK(TagDataPK tagDataPK) { TagData obj = (TagData) criteria.uniqueResult(); return obj; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("error in getSingleDataByPK for crisisCode : " + tagDataPK.getCrisisCode() + + " attributeCode : " + tagDataPK.getAttributeCode() + + " labelCode : " +tagDataPK.getLabelCode() + "timestamp : " + tagDataPK.getTimestamp()); } return null; } @@ -119,8 +118,7 @@ public List getDataByCrisis(String crisisCode) { //System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("Error in getDataByCrisis for crisis : " + crisisCode); } return null; } @@ -154,8 +152,8 @@ public Map getTagCountByCrisisGranularity(String crisisCode, Long } return data; } catch (HibernateException e) { - //logger.error("exception: ", e); - e.printStackTrace(); + logger.error("Error in getTagCountByCrisisGranularity for crisisCode : " + crisisCode + + " and timestamp : " + timestamp, e); } return null; } @@ -171,8 +169,7 @@ public List getAttributesForCrisis(String crisisCode) { //System.out.println("fetched attributes list size = " + (attributesList != null ? attributesList.size() : "null")); return attributesList; } catch (Exception e) { - e.printStackTrace(); - logger.error("exception: ", e); + logger.error("exception in getAttributesForCrisis for crisisCode : " + crisisCode, e); return null; } } @@ -187,8 +184,7 @@ public List getGranularitiesForCrisis(String crisisCode) { System.out.println("fetched attributes list size = " + (gList != null ? gList.size() : "null")); return gList; } catch (Exception e) { - e.printStackTrace(); - logger.error("exception: ", e); + logger.error("exception in getGranularitiesForCrisis for crisisCode : " + crisisCode, e); return null; } @@ -213,8 +209,8 @@ public List getDataByCrisisGranularity(String crisisCode, Long timestam System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataByCrisisGranularity for crisisCode : " + crisisCode + + " granularity : " + granularity, e); } return null; } @@ -234,14 +230,13 @@ public List getDataByCrisisAttributeLabel(String crisisCode, String att .add(Restrictions.eq("labelCode", labelCode)); } criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataByCrisisAttributeLabel for crisisCode : " + crisisCode + + " attributeCode : " + attributeCode + " labelCode : " + labelCode, e); } return null; } @@ -261,8 +256,9 @@ public List getDataByCrisisAttributeLabelGranularity(String crisisCode, System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataByCrisisAttributeLabelGranularity for crisisCode : " + crisisCode + + " attributeCode : " + attributeCode + " labelCode : " + labelCode + + " granularity : " + granularity, e); } return null; } @@ -287,8 +283,9 @@ public List getDataByGranularityInTimeWindow(String crisisCode, String System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataByGranularityInTimeWindow for crisisCode : " + crisisCode + + " attributeCode : " + attributeCode + " timestamp : " + timestamp + + " granularity : " + granularity, e); } return null; } @@ -313,8 +310,8 @@ public List getDataAfterTimestamp(String crisisCode, String attributeCo System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataAfterTimestampGranularity for crisisCode : " + crisisCode + + " attributeCode : " + attributeCode + " timestamp : " + timestamp, e); } return null; } @@ -339,8 +336,9 @@ public List getDataAfterTimestampGranularity(String crisisCode, String System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataAfterTimestampGranularity for crisisCode : " + crisisCode + + " attributeCode : " + attributeCode + " timestamp : " + timestamp + + " granularity : " + granularity, e); } return null; } @@ -365,8 +363,8 @@ public List getDataBeforeTimestamp(String crisisCode, String attributeC System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataBeforeTimestamp for crisisCode : " + crisisCode + + " attributeCode : " + attributeCode + " timestamp : " + timestamp, e); } return null; } @@ -393,8 +391,9 @@ public List getDataBeforeTimestampGranularity(String crisisCode, String System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataBeforeTimestampGranularity for crisisCode : " + crisisCode + + " attributeCode : " + attributeCode + " timestamp : " + timestamp + + " granularity : " + granularity, e); } return null; } @@ -420,8 +419,9 @@ public List getDataInInterval(String crisisCode, String attributeCode, System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataInInterval for crisisCode : " + crisisCode + + " attributeCode : " + attributeCode + " timestamp1 : " + timestamp1 + + " timestamp2 : " + timestamp2, e); } return null; } @@ -465,8 +465,9 @@ public List getDataInIntervalWithGranularity(String crisisCode, String System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { - //logger.error(elog.toStringException(e)); - e.printStackTrace(); + logger.error("exception in getDataInIntervalWithGranularity for crisisCode : " + crisisCode + + " attributeCode : " + attributeCode + " timestamp1 : " + timestamp1 + + " timestamp2 : " + timestamp2 + " granularity : " + granularity, e); } return null; } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java index 66eccf403..8052bbf2f 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java @@ -2,15 +2,12 @@ package qa.qcri.aidr.analysis.service; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; -import javax.annotation.PostConstruct; import javax.ejb.EJB; -import javax.ejb.Local; import javax.ejb.Stateless; import net.minidev.json.JSONObject; @@ -72,7 +69,6 @@ public JSONObject getTagCountSumForAllAttributesFromTime(String crisisCode, Long try { json.put("data", tagCountMap); dataSet.put(attribute, json); - System.out.println("Added json: " + json.toJSONString()); } catch (Exception e) { json = JsonResponse.addError(json); logger.error("Error in serializing fetched tag count data", e); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java index 50c5c585c..759fe71a0 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java @@ -1,8 +1,6 @@ package qa.qcri.aidr.analysis.stat; import java.util.List; -import java.util.Map; -import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; public class ConfDataMapRecord extends MapRecord { diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurationProperty.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurationProperty.java index 690a073e9..ddaf68138 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurationProperty.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurationProperty.java @@ -14,7 +14,7 @@ */ public enum AnalyticsConfigurationProperty implements ConfigurationProperty { - GRANULARITY("granularity"); + GRANULARITY("granularity"), TAGGER_REST_URI("TAGGER_REST_URI"); private final String configurationProperty; diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurator.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurator.java index 0977477d4..1a9e207e6 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurator.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurator.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import org.apache.log4j.Logger; @@ -18,7 +17,7 @@ public class AnalyticsConfigurator extends BaseConfigurator { private static final Logger logger = Logger .getLogger(AnalyticsConfigurator.class); - public static String configLoadFileName = "granularity.properties"; + public static String configLoadFileName = "config.properties"; private static final AnalyticsConfigurator instance = new AnalyticsConfigurator(); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsErrorLog.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsErrorLog.java new file mode 100644 index 000000000..f41ebee8b --- /dev/null +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsErrorLog.java @@ -0,0 +1,46 @@ +/** + * + */ +package qa.qcri.aidr.analysis.utils; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Form; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.log4j.Logger; +import org.glassfish.jersey.jackson.JacksonFeature; + +/** + * @author Latika + * TODO centralize in all modules + */ +public class AnalyticsErrorLog { + + private static Logger logger = Logger.getLogger(AnalyticsErrorLog.class); + + public static void sendErrorMail(String code, String errorMsg) { + Response clientResponse = null; + Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build(); + try { + WebTarget webResource = client.target(AnalyticsConfigurator.getInstance() + .getProperty(AnalyticsConfigurationProperty.TAGGER_REST_URI) + "/misc/sendErrorEmail"); + + Form form = new Form(); + form.param("module", "AIDRAnalytics"); + form.param("code", code); + form.param("description", errorMsg); + + clientResponse = webResource.request().post( + Entity.entity(form,MediaType.APPLICATION_FORM_URLENCODED),Response.class); + if (clientResponse.getStatus() != 200) { + logger.warn("Couldn't contact AIDRTaggerAPI for sending error message"); + } + } catch (Exception e) { + logger.error("Error in contacting AIDRTaggerAPI: " + clientResponse); + } + } +} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java index 5b48e6a1a..1367361e7 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java @@ -26,18 +26,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; - - - - - import org.apache.log4j.Logger; - - - - - import qa.qcri.aidr.analysis.stat.ConfDataMapRecord; import qa.qcri.aidr.analysis.stat.MapRecord; import qa.qcri.aidr.analysis.stat.TagDataMapRecord; @@ -45,7 +35,6 @@ import qa.qcri.aidr.common.filter.ClassifiedFilteredTweet; import qa.qcri.aidr.common.filter.NominalLabel; import qa.qcri.aidr.common.redis.LoadShedder; -import qa.qcri.aidr.analysis.utils.OutputConfigurator; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPubSub; import redis.clients.jedis.exceptions.JedisConnectionException; @@ -130,9 +119,8 @@ public void initiateChannelBufferManager(final String channelRegEx) { } catch (JedisConnectionException e) { subscriberJedis = null; isConnected = false; - logger.error("Fatal error! Couldn't establish connection to REDIS!"); - logger.error("exception", e); - e.printStackTrace(); + logger.error("Fatal error! Couldn't establish connection to REDIS!", e); + AnalyticsErrorLog.sendErrorMail("Redis", e.getMessage()); } if (isConnected) { aidrSubscriber = new RedisSubscriber(); @@ -143,11 +131,10 @@ public void initiateChannelBufferManager(final String channelRegEx) { isSubscribed = true; aidrSubscriber.setChannelName(channelRegEx); logger.info("Created pattern subscription for pattern: " + channelRegEx); - System.out.println("Created pattern subscription for pattern: " + channelRegEx); } catch (Exception e) { isSubscribed = false; - logger.error("Fatal exception occurred attempting subscription: " + e.toString()); - logger.error("exception", e); + logger.error("Fatal exception occurred attempting subscription: " + e.toString(), e); + AnalyticsErrorLog.sendErrorMail("Redis", e.getMessage()); } if (isSubscribed) { subscribedChannels = new HashSet(); @@ -248,7 +235,6 @@ public void manageChannelBuffers(final String subscriptionPattern, final String } } catch (Exception e) { logger.error("Exception", e); - e.printStackTrace(); } } // Periodically check if any channel is down - if so, delete all @@ -294,8 +280,7 @@ public boolean isChannelPresent(String channelName) { //logger.info("Checking channelName: " + channelName + ", result = " + subscribedChannels.containsKey(channelName) + ", with message count = " + subscribedChannels.get(channelName).getCurrentMsgCount()); return (subscribedChannels != null) ? subscribedChannels.contains(channelName) : false; } catch (Exception e) { - logger.error(Thread.currentThread().getName() + ":: Unable to check if channel present: " + channelName); - e.printStackTrace(); + logger.error(Thread.currentThread().getName() + ":: Unable to check if channel present: " + channelName, e); return false; } } @@ -310,8 +295,7 @@ public Set getActiveChannelsList() { Set channelSet = (subscribedChannels != null && !subscribedChannels.isEmpty()) ? subscribedChannels : null; return channelSet; } catch (Exception e) { - logger.error("Unable to fetch list of active channels"); - e.printStackTrace(); + logger.error("Unable to fetch list of active channels", e); return null; } } @@ -328,9 +312,7 @@ public void run() { // Execute the blocking REDIS subscription call subscriberJedis.psubscribe(aidrSubscriber, channelRegEx); } catch (JedisConnectionException e) { - logger.error("AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx); - System.out.println("[subscribeToChannel] AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx); - e.printStackTrace(); + logger.error("AIDR Predict Channel pSubscribing failed for channel = " + channelRegEx, e); stopSubscription(); Thread.currentThread().interrupt(); } @@ -348,8 +330,7 @@ private void stopSubscription() { logger.info("Unsubscribed from channel pattern: " + CHANNEL_PREFIX_STRING); } } catch (JedisConnectionException e) { - logger.error("Connection to REDIS seems to be lost!"); - logger.error("exception", e); + logger.error("Connection to REDIS seems to be lost!", e); } try { if (jedisConn != null && aidrSubscriber != null && subscriberJedis != null) { @@ -359,8 +340,7 @@ private void stopSubscription() { System.out.println("[stopSubscription] Stopsubscription completed..."); } } catch (Exception e) { - logger.error("Failed to return Jedis resource"); - logger.error("exception", e); + logger.error("Failed to return Jedis resource", e); } logger.info("isShutDown initiated = " + shutdownFlag); if (!shutdownFlag) { @@ -387,7 +367,6 @@ private void attemptResubscription() { isSubscribed = false; isSetup = false; logger.error("Fatal exception occurred attempting subscription: " + e.toString()); - logger.error("exception", e); ++attempts; } } @@ -401,9 +380,8 @@ private boolean setupRedisConnection(final String channelRegEx) { } catch (JedisConnectionException e) { subscriberJedis = null; isConnected = false; - logger.error("Fatal error! Couldn't establish connection to REDIS!"); - System.out.println("Fatal error! Couldn't establish connection to REDIS!"); - logger.error("exception", e); + logger.error("Fatal error! Couldn't establish connection to REDIS!", e); + AnalyticsErrorLog.sendErrorMail("Redis", e.getMessage()); } if (isConnected) { aidrSubscriber = new RedisSubscriber(); @@ -418,8 +396,7 @@ private boolean setupRedisConnection(final String channelRegEx) { } catch (Exception e) { isSubscribed = false; logger.error("Fatal exception occurred attempting subscription: " + e.toString()); - System.out.println("Fatal exception occurred attempting subscription "); - logger.error("exception", e); + AnalyticsErrorLog.sendErrorMail("Redis", e.getMessage()); } } return false; @@ -447,7 +424,7 @@ void shutdownAndAwaitTermination() { executorServicePool.shutdownNow(); // Cancel currently executing tasks // Wait a while for tasks to respond to being cancelled if (!executorServicePool.awaitTermination(5, TimeUnit.SECONDS)) - logger.error("Executor Thread Pool did not terminate"); + logger.warn("Executor Thread Pool did not terminate"); } else { logger.info("All tasks completed post service shutdown"); } @@ -508,9 +485,7 @@ public void onPMessage(String pattern, String channel, String message) { System.out.println("Done putting message"); } } catch (Exception e) { - logger.error("Exception occurred, redisLoadShedder = " + redisLoadShedder + ", channel status: " + redisLoadShedder.containsKey(channel)); - System.out.println("Exception occurred, redisLoadShedder = " + redisLoadShedder + ", channel status: " + redisLoadShedder.containsKey(channel)); - logger.error("exception", e); + logger.error("Exception occurred, redisLoadShedder = " + redisLoadShedder + ", channel status: " + redisLoadShedder.containsKey(channel), e); } } diff --git a/aidr-analytics/src/main/resources/config.properties b/aidr-analytics/src/main/resources/config.properties index d51ae4be2..777af4d5a 100644 --- a/aidr-analytics/src/main/resources/config.properties +++ b/aidr-analytics/src/main/resources/config.properties @@ -30,3 +30,9 @@ MAX_MESSAGES_COUNT=1000 TAGGER_CHANNEL_BASENAME=${TAGGER_CHANNEL_BASENAME} PERSISTER_LOAD_LIMIT=${PERSISTER_LOAD_LIMIT} PERSISTER_LOAD_CHECK_INTERVAL_MINUTES=${PERSISTER_LOAD_CHECK_INTERVAL_MINUTES} + +# Granularity settings +granularity=10s, 1m, 2m + +# Tagger URI +TAGGER_REST_URI=http://${TAGGER_API_HOST}:${TAGGER_API_PORT}/${TAGGER_API_WEB_APP_CONTEXT}/rest diff --git a/aidr-analytics/src/main/resources/granularity.properties b/aidr-analytics/src/main/resources/granularity.properties deleted file mode 100644 index 861316a7e..000000000 --- a/aidr-analytics/src/main/resources/granularity.properties +++ /dev/null @@ -1,11 +0,0 @@ -# -# Analytics Module Specific Configurations. -# - - -############################# -# OPTIONAL configuration -# Values can be left at their default values -############################# - -granularity=10s, 1m, 2m diff --git a/aidr-analytics/src/main/resources/log4j.properties b/aidr-analytics/src/main/resources/log4j.properties index fa82ee3b9..3df79e7e7 100644 --- a/aidr-analytics/src/main/resources/log4j.properties +++ b/aidr-analytics/src/main/resources/log4j.properties @@ -1,6 +1,6 @@ # Root logger option log=${LOG_LOCATION} -log4j.rootLogger=INFO, file +log4j.rootLogger=${LOG_LEVEL}, file # Direct log messages to a log file log4j.appender.file=org.apache.log4j.RollingFileAppender From 0c938141676984bf90f66ec087f112acac86fa61 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Fri, 31 Jul 2015 16:55:17 +0530 Subject: [PATCH 27/44] Added logs in AIDR-Collector. --- .../api/CollectorManageResource.java | 2 +- .../aidr/collector/api/SMSCollectorAPI.java | 4 +- .../collector/api/TwitterCollectorAPI.java | 8 +- .../qa/qcri/aidr/collector/beans/AIDR.java | 4 - .../aidr/collector/beans/CollectorStatus.java | 3 - .../aidr/collector/beans/ResponseWrapper.java | 79 ++++++++----------- .../collector/global/ApplicationConfig.java | 6 +- .../init/CollectorStartStopController.java | 8 +- 8 files changed, 49 insertions(+), 65 deletions(-) diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/CollectorManageResource.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/CollectorManageResource.java index 1c2018435..5496a949c 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/CollectorManageResource.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/CollectorManageResource.java @@ -132,7 +132,7 @@ private String runCollection(CollectionTask collection) { logger.info("Fetcher Response: " + jsonResponse); return jsonResponse; } catch (Exception e) { - logger.error("Could not start collection"); + logger.error("Could not start collection"+collection.getCollectionCode()); return "Could not start collection"; } } diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/SMSCollectorAPI.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/SMSCollectorAPI.java index 875ea65f3..f5b8677e2 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/SMSCollectorAPI.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/SMSCollectorAPI.java @@ -28,6 +28,7 @@ import qa.qcri.aidr.collector.collectors.JedisPublisher; import qa.qcri.aidr.collector.utils.CollectorConfigurationProperty; import qa.qcri.aidr.collector.utils.CollectorConfigurator; +import qa.qcri.aidr.collector.utils.CollectorErrorLog; import qa.qcri.aidr.collector.utils.GenericCache; import qa.qcri.aidr.common.redis.LoadShedder; @@ -135,8 +136,8 @@ public void startPersister(String collectionCode) { logger.info(collectionCode + ": Collector persister response = " + jsonResponse); } catch (RuntimeException e) { logger.error(collectionCode + ": Could not start persister. Is persister running?"); + CollectorErrorLog.sendErrorMail(collectionCode, "Unable to start persister."); } catch (UnsupportedEncodingException e) { - // TODO Auto-generated catch block logger.error(collectionCode + ": Unsupported Encoding scheme used"); } } @@ -152,7 +153,6 @@ public void stopPersister(String collectionCode) { } catch (RuntimeException e) { logger.error(collectionCode + ": Could not stop persister. Is persister running?"); } catch (UnsupportedEncodingException e) { - // TODO Auto-generated catch block logger.error(collectionCode + ": Unsupported Encoding scheme used"); } } diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/TwitterCollectorAPI.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/TwitterCollectorAPI.java index 2a399dee9..100f047d4 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/TwitterCollectorAPI.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/api/TwitterCollectorAPI.java @@ -28,6 +28,7 @@ import qa.qcri.aidr.collector.collectors.TwitterStreamTracker; import qa.qcri.aidr.collector.utils.CollectorConfigurator; import qa.qcri.aidr.collector.utils.CollectorConfigurationProperty; +import qa.qcri.aidr.collector.utils.CollectorErrorLog; import qa.qcri.aidr.collector.utils.GenericCache; /** @@ -240,8 +241,8 @@ public void startCollectorPersister(String collectionCode) { logger.info(collectionCode + ": Collector persister response = " + jsonResponse); } catch (RuntimeException e) { logger.error(collectionCode + ": Could not start persister. Is persister running?", e); + CollectorErrorLog.sendErrorMail(collectionCode, "Unable to start persister. Is persister running"); } catch (UnsupportedEncodingException e) { - // TODO Auto-generated catch block logger.error(collectionCode + ": Unsupported Encoding scheme used"); } } @@ -258,6 +259,7 @@ private void startPersister(String collectionCode) { logger.info(collectionCode + ": Collector persister response = " + jsonResponse); } catch (RuntimeException e) { logger.error(collectionCode + ": Could not start persister. Is persister running?", e); + CollectorErrorLog.sendErrorMail(collectionCode, "Unable to start persister. Is persister running"); } catch (UnsupportedEncodingException e) { logger.error(collectionCode + ": Unsupported Encoding scheme used"); } @@ -275,7 +277,6 @@ public void stopCollectorPersister(String collectionCode) { } catch (RuntimeException e) { logger.error(collectionCode + ": Could not stop persister. Is persister running?", e); } catch (UnsupportedEncodingException e) { - // TODO Auto-generated catch block logger.error(collectionCode + ": Unsupported Encoding scheme used"); } } @@ -291,7 +292,6 @@ public void stopPersister(String collectionCode) { } catch (RuntimeException e) { logger.error(collectionCode + ": Could not stop persister. Is persister running?", e); } catch (UnsupportedEncodingException e) { - // TODO Auto-generated catch block logger.error(collectionCode + ": Unsupported Encoding scheme used"); } } @@ -309,7 +309,6 @@ public void startTaggerPersister(String collectionCode) { } catch (RuntimeException e) { logger.error(collectionCode + ": Could not start persister. Is persister running?", e); } catch (UnsupportedEncodingException e) { - // TODO Auto-generated catch block logger.error(collectionCode + ": Unsupported Encoding scheme used"); } } @@ -326,7 +325,6 @@ public void stopTaggerPersister(String collectionCode) { } catch (RuntimeException e) { logger.error(collectionCode + ": Could not stop persister. Is persister running?", e); } catch (UnsupportedEncodingException e) { - // TODO Auto-generated catch block logger.error(collectionCode + ": Unsupported Encoding scheme used"); } } diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/AIDR.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/AIDR.java index 7ba6c166b..1fc468b7f 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/AIDR.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/AIDR.java @@ -66,8 +66,4 @@ public void setDoctype(String doctype) { this.doctype = doctype; } - - - - } diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/CollectorStatus.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/CollectorStatus.java index b1980e40a..ea5dc19e0 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/CollectorStatus.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/CollectorStatus.java @@ -81,7 +81,4 @@ public int getRunningCollectionsCount() { public void setRunningCollectionsCount(int runningCollectionsCount) { this.runningCollectionsCount = runningCollectionsCount; } - - - } diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/ResponseWrapper.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/ResponseWrapper.java index b18cd5697..fec5e8177 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/ResponseWrapper.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/beans/ResponseWrapper.java @@ -18,58 +18,49 @@ */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "responseWrapper", propOrder = { - "statusCode", - "message", - "dataObject", - - }) + "statusCode", + "message", + "dataObject", + +}) @XmlRootElement(name = "responseWrapper") @JsonSerialize(include = Inclusion.NON_DEFAULT) public class ResponseWrapper implements Serializable{ - - protected String statusCode; - protected String message; - protected Object dataObject; - - - public ResponseWrapper() { - } - - - - public String getStatusCode() { - return statusCode; - } - - /** - * @param statusCode the statusCode to set - */ - public void setStatusCode(String statusCode) { - this.statusCode = statusCode; - } - public Object getDataObject() { - return dataObject; - } - public void setDataObject(Object dataObject) { - this.dataObject = dataObject; - } + /** + * + */ + private static final long serialVersionUID = -6615942283542314788L; + protected String statusCode; + protected String message; + protected Object dataObject; - public String getMessage() { - return message; - } + public ResponseWrapper() { + } - public void setMessage(String message) { - this.message = message; - } + public String getStatusCode() { + return statusCode; + } - /** - * @return the statusCode - */ - + /** + * @param statusCode the statusCode to set + */ + public void setStatusCode(String statusCode) { + this.statusCode = statusCode; + } + public Object getDataObject() { + return dataObject; + } - + public void setDataObject(Object dataObject) { + this.dataObject = dataObject; + } - + public String getMessage() { + return message; + } + public void setMessage(String message) { + this.message = message; + } } diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/global/ApplicationConfig.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/global/ApplicationConfig.java index b20203726..e4dc420b5 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/global/ApplicationConfig.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/global/ApplicationConfig.java @@ -9,6 +9,7 @@ import javax.ws.rs.ApplicationPath; import javax.ws.rs.core.Application; +import org.apache.log4j.Logger; import org.glassfish.jersey.jackson.JacksonFeature; /** @@ -17,10 +18,11 @@ */ @ApplicationPath("/webresources") public class ApplicationConfig extends Application { - + private static Logger logger = Logger + .getLogger(ApplicationConfig.class); @Override public Set> getClasses() { - System.out.println("In ApplicationConfig: registering REST APIs"); + logger.info("In ApplicationConfig: registering REST APIs"); Set> resources = new java.util.HashSet>(); resources.add(JacksonFeature.class); addRestResourceClasses(resources); diff --git a/aidr-collector/src/main/java/qa/qcri/aidr/collector/init/CollectorStartStopController.java b/aidr-collector/src/main/java/qa/qcri/aidr/collector/init/CollectorStartStopController.java index 74363fbb4..a7c1410de 100644 --- a/aidr-collector/src/main/java/qa/qcri/aidr/collector/init/CollectorStartStopController.java +++ b/aidr-collector/src/main/java/qa/qcri/aidr/collector/init/CollectorStartStopController.java @@ -39,7 +39,7 @@ public CollectorStartStopController() { @PostConstruct private void startup() { // Startup tasks go here - System.out.println("AIDR-Collector: Starting up..."); + //System.out.println("AIDR-Collector: Starting up..."); logger.info("AIDR-Collector: Starting up..."); // task todo DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); @@ -54,7 +54,7 @@ private void startup() { @PreDestroy private void shutdown() { // Shutdown tasks go here - System.out.println("AIDR-Collector: Shutting Down..."); + //System.out.println("AIDR-Collector: Shutting Down..."); logger.info("AIDR-Collector: Shutting Down..."); List collections = GenericCache.getInstance() .getAllRunningCollectionTasks(); @@ -66,11 +66,11 @@ private void shutdown() { try { Thread.sleep(1000); } catch (InterruptedException e) { - logger.warn("Error : " + e.getMessage()); + logger.warn("Error while PreDestroying the collector: " + e.getMessage()); } } - System.out.println("AIDR-Collector: Shutdown procedure completed."); + //System.out.println("AIDR-Collector: Shutdown procedure completed."); logger.info("AIDR-Collector: Shutdown procedure completed."); } From 3632f2cbe0c0669314597fe1ace32a0f164d897d Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Fri, 31 Jul 2015 17:09:59 +0530 Subject: [PATCH 28/44] Commit for #97809356 : aidr-task-manager logging --- .../analysis/api/WriteStatisticsData.java | 3 - ...ConfidenceStatisticsResourceFacadeImp.java | 9 +-- .../TagDataStatisticsResourceFacadeImp.java | 27 ------- .../service/GetTagDataStatisticsService.java | 7 -- .../aidr/analysis/stat/ConfDataMapRecord.java | 1 - .../aidr/analysis/stat/TagDataMapRecord.java | 1 - .../analysis/utils/ChannelBufferManager.java | 16 +--- .../aidr/task/ejb/bean/TaskManagerBean.java | 76 +++++-------------- 8 files changed, 20 insertions(+), 120 deletions(-) diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java index f8b3fc7a3..4dc6391b4 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -156,7 +156,6 @@ private ReturnCode writeOutputDataToConfDataDB(Long granularity, Long timestamp) ConfidenceData f = new ConfidenceData(key.getCrisisCode(), timestamp, granularity, key.getAttributeCode(), key.getLabelCode(), Integer.parseInt(((ConfCounterKey) key).getBinNumber()), fCount.getCount(granularity)); if (fCount.getCount(granularity) > 0) { - System.out.println("Will attempt persistence of conf key: " + key.toString()); confDataEJB.writeData(f); } fCount.resetCount(granularity); @@ -178,7 +177,6 @@ private class WriterThread implements Runnable { @Override public void run() { logger.info("Started aidr-analytics writer thread: " + t.getName()); - System.out.println("Started aidr-analytics writer thread: " + t.getName()); Map lastTagDataWriteTime = new TreeMap(); Map lastConfDataWriteTime = new TreeMap(); for (Long g : granularityList) { @@ -196,7 +194,6 @@ public void run() { //logger.info("retVal = " + retVal); if (ReturnCode.SUCCESS.equals(retVal)) { //logger.info("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); - //System.out.println("Successfully wrote for granularity: " + granularity + " at time = " + new Date(currentTime)); } } if (0 == lastConfDataWriteTime.get(granularity) || (currentTime - lastConfDataWriteTime.get(granularity)) >= granularity) { diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java index eac117ce2..e3600a2a2 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/ConfidenceStatisticsResourceFacadeImp.java @@ -74,16 +74,11 @@ public Session getCurrentSession() { public ReturnCode writeData(ConfidenceData confData) { try { em.persist(confData); - System.out.println("Success in persisting conf data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() - + ", " + confData.getLabelCode() + ", " + confData.getTimestamp() + ", " + confData.getGranularity() - + ", " + confData.getBin() + ": " + confData.getCount()); return ReturnCode.SUCCESS; } catch (Exception e) { - System.out.println("Failure in persisting conf data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + logger.error("Failure in persisting conf data for: " + confData.getCrisisCode() + ", " + confData.getAttributeCode() + ", " + confData.getLabelCode() + ", " + confData.getTimestamp() + ", " + confData.getGranularity() - + ", " + confData.getBin() + ": " + confData.getCount()); - logger.error("exception", e); - e.printStackTrace(); + + ", " + confData.getBin() + ": " + confData.getCount(), e); return ReturnCode.ERROR; } } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java index a5a17365f..720fd020f 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/facade/imp/TagDataStatisticsResourceFacadeImp.java @@ -95,7 +95,6 @@ public TagData getSingleDataByPK(TagDataPK tagDataPK) { .add(Restrictions.eq("attributeCode", tagDataPK.getAttributeCode())) .add(Restrictions.eq("labelCode", tagDataPK.getLabelCode())); criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { TagData obj = (TagData) criteria.uniqueResult(); return obj; @@ -112,10 +111,8 @@ public TagData getSingleDataByPK(TagDataPK tagDataPK) { public List getDataByCrisis(String crisisCode) { Criteria criteria = getCurrentSession().createCriteria(TagData.class); criteria.add(Restrictions.eq("crisisCode", crisisCode)); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - //System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("Error in getDataByCrisis for crisis : " + crisisCode); @@ -139,15 +136,12 @@ public Map getTagCountByCrisisGranularity(String crisisCode, Long .add(Projections.sum("count").as("count"))) ).list(); - //System.out.println("fetched list size = " + (countList != null ? countList.size() : "null")); Map data = new HashMap(); for (int i = 0; i < countList.size();i++) { Object[] temp = (Object[]) countList.get(i); Long g = ((Number) temp[0]).longValue(); Long count = ((Number) temp[1]).longValue(); data.put(g.toString(), count); - //System.out.println("Fetched {" + temp[0] + ", " + temp[1] + "}"); - //System.out.println("1 --> " + g.getClass() + ", 2 --> " + temp[1].getClass()); } return data; @@ -166,7 +160,6 @@ public List getAttributesForCrisis(String crisisCode) { List attributesList = (List) getCurrentSession().createCriteria(TagData.class) .add(Restrictions.eq("crisisCode", crisisCode)) .setProjection(Projections.distinct(Projections.property("attributeCode"))).list(); - //System.out.println("fetched attributes list size = " + (attributesList != null ? attributesList.size() : "null")); return attributesList; } catch (Exception e) { logger.error("exception in getAttributesForCrisis for crisisCode : " + crisisCode, e); @@ -181,7 +174,6 @@ public List getGranularitiesForCrisis(String crisisCode) { List gList = (List) getCurrentSession().createCriteria(TagData.class) .add(Restrictions.eq("crisisCode", crisisCode)) .setProjection(Projections.distinct(Projections.property("granularity"))).list(); - System.out.println("fetched attributes list size = " + (gList != null ? gList.size() : "null")); return gList; } catch (Exception e) { logger.error("exception in getGranularitiesForCrisis for crisisCode : " + crisisCode, e); @@ -203,10 +195,8 @@ public List getDataByCrisisGranularity(String crisisCode, Long timestam .add(Restrictions.eq("granularity", granularity)); } criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataByCrisisGranularity for crisisCode : " + crisisCode @@ -232,7 +222,6 @@ public List getDataByCrisisAttributeLabel(String crisisCode, String att criteria.add(criterion); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataByCrisisAttributeLabel for crisisCode : " + crisisCode @@ -250,10 +239,8 @@ public List getDataByCrisisAttributeLabelGranularity(String crisisCode, .add(Restrictions.eq("labelCode", labelCode)) .add(Restrictions.eq("granularity", granularity)); criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataByCrisisAttributeLabelGranularity for crisisCode : " + crisisCode @@ -277,10 +264,8 @@ public List getDataByGranularityInTimeWindow(String crisisCode, String } criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataByGranularityInTimeWindow for crisisCode : " + crisisCode @@ -304,10 +289,8 @@ public List getDataAfterTimestamp(String crisisCode, String attributeCo } criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataAfterTimestampGranularity for crisisCode : " + crisisCode @@ -330,10 +313,8 @@ public List getDataAfterTimestampGranularity(String crisisCode, String .add(Restrictions.eq("labelCode", labelCode)); } criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataAfterTimestampGranularity for crisisCode : " + crisisCode @@ -357,10 +338,8 @@ public List getDataBeforeTimestamp(String crisisCode, String attributeC } criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataBeforeTimestamp for crisisCode : " + crisisCode @@ -385,10 +364,8 @@ public List getDataBeforeTimestampGranularity(String crisisCode, String } criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataBeforeTimestampGranularity for crisisCode : " + crisisCode @@ -413,10 +390,8 @@ public List getDataInInterval(String crisisCode, String attributeCode, .add(Restrictions.eq("labelCode", labelCode)); } criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataInInterval for crisisCode : " + crisisCode @@ -459,10 +434,8 @@ public List getDataInIntervalWithGranularity(String crisisCode, String .add(Restrictions.le("timestamp", timestamp2)); } criteria.add(criterion); - System.out.println("Formed criteria: " + criteria.toString()); try { List objList = (List) criteria.list(); - System.out.println("fetched list size = " + (objList != null ? objList.size() : "null")); return objList; } catch (HibernateException e) { logger.error("exception in getDataInIntervalWithGranularity for crisisCode : " + crisisCode diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java index 8052bbf2f..20c3b53e9 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/service/GetTagDataStatisticsService.java @@ -52,7 +52,6 @@ public JSONObject getTagCountSumForAllAttributesFromTime(String crisisCode, Long jsonList.put("attributesList", attributesList); Map dataSet = new HashMap(); for (String attribute: attributesList) { - System.out.println("For attribute: " + attribute); tagDataList.addAll(tagDataEJB.getDataAfterTimestampGranularity(crisisCode, attribute, null, startTime, granularity)); // Now the real work - count and send response JSONObject json = new JsonResponse().getNewJsonResponseObject(crisisCode, attribute, granularity, startTime, null); @@ -90,7 +89,6 @@ public JSONObject getTagCountSumByGranularity(String crisisCode, Long startTime) json.put("granularities", gList); json.put("data", countList); } - //System.out.println("Returned json: " + json.toJSONString()); return json; } @@ -104,8 +102,6 @@ public JSONObject getTagCountSumByGranularity(String crisisCode, Long startTime) */ public JSONObject getTagCountSumFromTime(String crisisCode, String attributeCode, Long granularity, Long startTime) { - System.out.println("Received parameters: crisisCode = " + crisisCode + ", attributeCode = " + attributeCode + ", granularity = " + granularity - + ", startTime = " + startTime + ", local EJB = " + tagDataEJB); // First get the list of data points from DB List tagDataList = tagDataEJB.getDataAfterTimestampGranularity(crisisCode, attributeCode, null, startTime, granularity); @@ -204,7 +200,6 @@ public JSONObject getTagCountTimeSeries(String crisisCode, String attributeCode, tagsList.add(TagCountDTOHelper.convertTagDataToDTO(t)); } } - //System.out.println("Finished creating Map of timestamp versus TagCountDTO list"); // Now convert the above time series data Map to DTO object for response List timeWindowDTOList = new ArrayList(); try { @@ -212,10 +207,8 @@ public JSONObject getTagCountTimeSeries(String crisisCode, String attributeCode, TimeWindowTagCountDTO timeWindowDTO = TimeWindowTagCountDTOHelper.convertTagCountDTOListToDTO(key, tagCountMap.get(key)); timeWindowDTOList.add(timeWindowDTO); } - //System.out.println("Finished creating TimeWindowTagCountDTO list"); TagCountSeriesDTO dto = TagCountSeriesDTOHelper.convertTimeWindowTagCountDTOListToDTO(crisisCode, attributeCode, granularity, timeWindowDTOList); json.put("data", dto); - //System.out.println("Finished creating json object: " + json); return json; } catch (Exception e) { json = JsonResponse.addError(json); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java index 759fe71a0..0c1797a77 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/ConfDataMapRecord.java @@ -6,7 +6,6 @@ public class ConfDataMapRecord extends MapRecord { public ConfDataMapRecord(List granularityList) { - System.out.println("Conf data granularityList: " + granularityList); this.count = new ConcurrentHashMap(); for (Long g: granularityList) { this.count.put(g, 0); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java index 94bc6c987..66ea942a4 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/stat/TagDataMapRecord.java @@ -19,7 +19,6 @@ public void setFirstEntryTime(Long firstEntryTime) { public TagDataMapRecord(List granularityList) { this.count = new ConcurrentHashMap(); - System.out.println("Tag data granulairtyList: " + granularityList); for (Long g: granularityList) { this.count.put(g, 0); } diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java index 1367361e7..ea06c6415 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java @@ -105,8 +105,6 @@ public void initiateChannelBufferManager(final String channelRegEx) { channelMap = new ConcurrentHashMap(); logger.info("Initializing channel buffer manager."); - System.out.println("[ChannelBufferManager] Initializing channel buffer manager with values: <" + redisHost + ", " + redisPort - + ", " + PERSISTER_LOAD_CHECK_INTERVAL_MINUTES + ", " + PERSISTER_LOAD_LIMIT + ">"); bufferSize = -1; executorServicePool = Executors.newCachedThreadPool(); //Executors.newFixedThreadPool(10); // max number of threads @@ -194,14 +192,13 @@ public void manageChannelBuffers(final String subscriptionPattern, final String logger.error("Something terribly wrong! Fatal error in: " + channelName); } else { if (!isChannelPresent(channelName)) { - System.out.println("New collection/channel found: " + channelName); + logger.info("New collection/channel found: " + channelName); subscribedChannels.add(channelName); } try { ClassifiedFilteredTweet classifiedTweet = new ClassifiedFilteredTweet().deserialize(receivedMessage); if (classifiedTweet != null && classifiedTweet.getNominalLabels() != null && !classifiedTweet.getNominalLabels().isEmpty()) { channelMap.putIfAbsent(classifiedTweet.getCrisisCode(), System.currentTimeMillis()); - //System.out.println("Found a valid classified tweet for collection: " + classifiedTweet.getCrisisCode() + ", tweet = " + receivedMessage); for (NominalLabel nb : classifiedTweet.getNominalLabels()) { if (nb.attribute_code != null && nb.label_code != null) { CounterKey tagDataKey = new CounterKey(classifiedTweet.getCrisisCode(), nb.attribute_code, nb.label_code); @@ -211,24 +208,17 @@ public void manageChannelBuffers(final String subscriptionPattern, final String TagDataMapRecord t = (TagDataMapRecord) tagDataMap.get(tagDataKey); t.incrementAllCounts(); tagDataMap.put(tagDataKey, t); - //System.out.println("Updated Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); } else { TagDataMapRecord t = new TagDataMapRecord(granularityList); tagDataMap.put(tagDataKey, t); - - //logger.info("New Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); - //System.out.println("New Tag map entry with key: " + tagDataKey.toString() + " value = " + tagDataMap.get(tagDataKey).toString()); } if (confDataMap.containsKey(confDataKey)) { ConfDataMapRecord f = (ConfDataMapRecord) confDataMap.get(confDataKey); f.incrementAllCounts(); confDataMap.put(confDataKey, f); - //System.out.println("Updated Conf map entry with key: " + confDataKey.toString() + " value = " + confDataMap.get(confDataKey).toString()); } else { ConfDataMapRecord t = new ConfDataMapRecord(granularityList); confDataMap.put(confDataKey, t); - //logger.info("[manageChannelBuffersWrapper] New Conf map entry with key: " + confDataKey + " value = " + confDataMap.get(confDataKey)); - //System.out.println("New Conf map entry with key: " + confDataKey.toString() + " value = " + confDataMap.get(confDataKey).toString()); } } } @@ -307,7 +297,6 @@ private void subscribeToChannel(final String channelRegEx) throws Exception { public void run() { Thread.currentThread().setName("ChannelBufferManager Redis subscription Thread"); logger.info("New thread <" + Thread.currentThread().getName() + "> created for subscribing to redis channel: " + channelRegEx); - System.out.println("New thread <" + Thread.currentThread().getName() + "> created for subscribing to redis channel: " + channelRegEx); try { // Execute the blocking REDIS subscription call subscriberJedis.psubscribe(aidrSubscriber, channelRegEx); @@ -337,7 +326,6 @@ private void stopSubscription() { jedisConn.returnJedis(subscriberJedis); subscriberJedis = null; logger.info("Stopsubscription completed..."); - System.out.println("[stopSubscription] Stopsubscription completed..."); } } catch (Exception e) { logger.error("Failed to return Jedis resource", e); @@ -408,7 +396,6 @@ public void close() { stopSubscription(); shutdownAndAwaitTermination(); logger.info("All done, fetch service has been shutdown..."); - System.out.println("[close] All done, fetch service has been shutdown..."); } @@ -482,7 +469,6 @@ public void onPMessage(String pattern, String channel, String message) { } if (redisLoadShedder.get(channel).canProcess(channel)) { manageChannelBuffers(pattern, channel, message); - System.out.println("Done putting message"); } } catch (Exception e) { logger.error("Exception occurred, redisLoadShedder = " + redisLoadShedder + ", channel status: " + redisLoadShedder.containsKey(channel), e); diff --git a/aidr-task-manager/src/main/java/qa/qcri/aidr/task/ejb/bean/TaskManagerBean.java b/aidr-task-manager/src/main/java/qa/qcri/aidr/task/ejb/bean/TaskManagerBean.java index 0f1b9eac6..d0fce1e2e 100644 --- a/aidr-task-manager/src/main/java/qa/qcri/aidr/task/ejb/bean/TaskManagerBean.java +++ b/aidr-task-manager/src/main/java/qa/qcri/aidr/task/ejb/bean/TaskManagerBean.java @@ -129,7 +129,7 @@ public Class getClassType() { @Override public long insertNewTask(T task) { if (task == null) { - logger.error("Attempting to insert NULL"); + logger.warn("Attempting to insert NULL"); return -1; } DocumentDTO doc = (DocumentDTO) task; @@ -139,7 +139,7 @@ public long insertNewTask(T task) { DocumentDTO savedDoc = remoteDocumentEJB.addDocument(doc); return savedDoc.getDocumentID(); } catch (Exception e) { - logger.error("Error in insertion"); + logger.error("Error in insertion : " + doc.getData()); } return -1; } @@ -156,12 +156,10 @@ public Long saveNewTask(T task, Long crisisID) { doc.setCrisisDTO(crisisDTO); doc.setHasHumanLabels(false); DocumentDTO savedDoc = remoteDocumentEJB.addDocument(doc); - //System.out.println("Saved to DB document: " + savedDoc.getDocumentID() + ", for crisis = " + savedDoc.getCrisisDTO().getCode()); //logger.info("Saved to DB document: " + savedDoc.getDocumentID() + ", for crisis = " + savedDoc.getCrisisDTO().getCode()); return savedDoc.getDocumentID(); } catch (Exception e) { - logger.error("Error in insertion"); - e.printStackTrace(); + logger.error("Error in insertion for crisisID : " + crisisID + " and doc : " + task.toString(), e); } return -1L; } @@ -199,7 +197,7 @@ public List saveNewTasks(List collection, Long crisisID) { } } } catch (Exception e) { - logger.error("Error in collection insertion"); + logger.error("Error in collection insertion for crisisID : " + crisisID); } } else { logger.warn("Attempting to insert NULL"); @@ -213,7 +211,6 @@ public int deleteTaskById(Long id) { try { DocumentDTO doc = remoteDocumentEJB.findDocumentByID(id); Integer result = remoteDocumentEJB.deleteNoLabelDocument(doc); - //System.out.println("[deleteTaskById] gotById document: " + document); return result; } catch(Exception e) { logger.error("Error in deletion"); @@ -225,9 +222,7 @@ public int deleteTaskById(Long id) { public int deleteTask(T task) { if (task != null) { try { - //System.out.println("[deleteTask] Going for deletion of fetched doc = " + ((Document) task).getDocumentID()); return remoteDocumentEJB.deleteNoLabelDocument((DocumentDTO) task); - //documentLocalEJB.delete((Document) task); } catch (Exception e) { logger.error("Error in deletion of task"); return 0; @@ -268,7 +263,6 @@ public int deleteTask(List collection) { dtoList = createDocumentDTOList((List) collection); } catch (PropertyNotSetException e) { logger.error("Unable to create DTO list.", e); - e.printStackTrace(); } try { @@ -331,7 +325,7 @@ public int deleteStaleTasks(String joinType, String joinTable, String joinColumn sortOrder, orderBy, maxTaskAge, scanInterval); return docDeleteCount; } catch (Exception e) { - logger.error("Error in deletion"); + logger.warn("Error in deletion"); return 0; } } @@ -350,11 +344,6 @@ public int truncateLabelingTaskBufferForCrisis(final long crisisID, final int ma Criterion aliasCriterion = (Restrictions.isNull(aliasTableKey)); docList = remoteDocumentEJB.getByCriteriaWithAliasByOrder(criterion, order, orderBy, null, aliasTable, aliasCriterion); - /* - System.out.println("Fetched docList = " + docList); - if (docList != null) { - System.out.println("Fetched size = " + docList.size()); - }*/ } catch (Exception e) { logger.error("Exception in fetching unassigned documents with hasHumaLabels=false"); return 0; @@ -496,9 +485,8 @@ public List getNewTaskCollection(Long crisisID, Integer count, Stri Criterion aliasCriterion = (Restrictions.isNull(aliasTableKey)); try { List docList = remoteDocumentEJB.getByCriteriaWithAliasByOrder(newCriterion, order, orderBy, count, aliasTable, aliasCriterion); - //System.out.println("[getNewTaskCollection] docList = " + docList); if (docList != null) { - System.out.println("[getNewTaskCollection] Fetched size = " + docList.size()); + logger.info("[getNewTaskCollection] Fetched size = " + docList.size()); return createDocumentDTOList(docList); } else { return null; @@ -710,9 +698,8 @@ public List getDocumentsForTagging(final Long crisisID, int count, try { users = remoteUsersEJB.getUserByName(userName); } catch (Exception e) { - System.err.println("[getDocumentsForTagging] Exception in finding user userName = " + userName + ". Aborting..."); + logger.error("[getDocumentsForTagging] Exception in finding user userName = " + userName + ". Aborting..."); logger.error("Exception", e); - e.printStackTrace(); return null; } if (users != null) { @@ -726,17 +713,13 @@ public List getDocumentsForTagging(final Long crisisID, int count, ++inCS; try { List dtoList = getNewTaskCollection(crisisID, fetchCount, "DESC", null); - System.out.println("[getDocumentsForTagging] inCS = " + inCS + ". For crisisID = " + crisisID + ", user = " + userName - + ", documents available: " + (dtoList != null ? dtoList.size() : "empty list")); if (dtoList != null) { fetchedSize = dtoList.size(); } int availableRequestSize = fetchedSize - remainingCount; - System.out.println("available = " + availableRequestSize + ", total = " + fetchedSize + ", remainingCount = " + remainingCount); if (availableRequestSize > 0) { count = Math.min(count, availableRequestSize); if (!dtoList.isEmpty() && count > 0) { - System.out.println("[getDocumentsForTagging] Going to insert task list of size = " + count + ", for userID: " + users.getUserID()); List assignList = new ArrayList(); assignList.addAll(dtoList.subList(0, count)); assignNewTaskToUser(assignList, users.getUserID()); @@ -751,13 +734,11 @@ public List getDocumentsForTagging(final Long crisisID, int count, } } catch (Exception e) { logger.error("Exception", e); - e.printStackTrace(); - //lock.unlock(); return null; } } } else { - System.err.println("[getDocumentsForTagging] No user found with userName = " + userName + ". Aborting..."); + logger.warn("[getDocumentsForTagging] No user found with userName = " + userName + ". Aborting..."); } return null; } @@ -773,7 +754,7 @@ public void assignNewTaskToUser(Long id, Long userId) throws Exception { throw new Exception("[assignNewTaskToUser] Couldn't do task assignment"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error in assignNewTaskToUser for id : " + id + " and userID : " + userId); } } @@ -782,6 +763,7 @@ public void assignNewTaskToUser(Long id, Long userId) throws Exception { public void assignNewTaskToUser(List collection, Long userId) throws Exception { int retVal = remoteTaskAssignmentEJB.insertTaskAssignment(collection, userId); if (retVal <= 0) { + logger.warn("Unable to undo task assignment"); throw new Exception("[assignNewTaskToUser] Couldn't do task assignment"); } } @@ -790,7 +772,7 @@ public void assignNewTaskToUser(List collection, Long userId) throw public void undoTaskAssignment(Map taskMap) throws Exception { int retVal = remoteTaskAssignmentEJB.undoTaskAssignment(taskMap); if (retVal <= 0) { - logger.error("Unable to undo task assignment"); + logger.warn("Unable to undo task assignment"); throw new Exception("[undoTaskAssignment] Couldn't undo task assignment"); } } @@ -799,7 +781,7 @@ public void undoTaskAssignment(Map taskMap) throws Exception { public void undoTaskAssignment(Long id, Long userId) throws Exception { int retVal = remoteTaskAssignmentEJB.undoTaskAssignment(id, userId); if (retVal <= 0) { - logger.error("Unable to undo task assignment"); + logger.warn("Unable to undo task assignment"); throw new Exception("[undoTaskAssignment] Couldn't undo task assignment"); } } @@ -925,11 +907,8 @@ public Object setTaskParameter(Class entityType, Long id, Map getAllUserByName(String name) { try { userList = remoteUsersEJB.getAllUsersByName(name); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getAllUserByName for name : " + name, e); } return userList; } @@ -1050,7 +1026,6 @@ public List getAllUserByName(String name) { public void saveDocumentNominalLabel(DocumentNominalLabelDTO documentNominalLabel) { try { DocumentNominalLabelDTO dto = remoteDocumentNominalLabelEJB.addDocument(documentNominalLabel); - System.out.println("Saved to DB document nominal label: " + dto.getIdDTO().getDocumentId() + ", with nominal label = " + dto.getIdDTO().getNominalLabelId()); logger.info("Saved to DB document nominal label: " + dto.getIdDTO().getDocumentId() + ", with nominal label = " + dto.getIdDTO().getNominalLabelId()); } catch (Exception e) { logger.error("Error in saving document nominal label : " + documentNominalLabel, e); @@ -1064,8 +1039,7 @@ public boolean foundDuplicateDocumentNominalLabel(DocumentNominalLabelDTO docume attMap.put("id.documentId", documentNominalLabel.getIdDTO().getDocumentId()); attMap.put("id.nominalLabelId", documentNominalLabel.getIdDTO().getNominalLabelId()); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.warn("Warning! duplication nominal label"); } DocumentNominalLabel obj = remoteDocumentNominalLabelEJB.getByCriterionID(Restrictions.allEq(attMap)); @@ -1115,10 +1089,8 @@ public List getHumanLabeledDocumentsByCrisisID(Long cri try { List docList = remoteDocumentEJB.getByCriteriaWithInnerJoinByOrder(criterion, "DESC", orderBy, count, aliasTable, aliasCriterion); if (docList != null) { - System.out.println("Fetched size = " + docList.size()); Set docSet = new TreeSet(new DocumentComparator()); docSet.addAll(docList); - System.out.println("Sizeof document collection set = " + docSet.size()); // First get all labels for the fetched documents labeledDocList = new ArrayList(); @@ -1184,12 +1156,9 @@ public List getHumanLabeledDocumentsByCrisisIDUserID(Lo .add(Restrictions.eq("documentNominalLabels.id.userId", userID)); try { List docList = remoteDocumentEJB.getByCriteriaWithInnerJoinByOrder(criterion, "DESC", orderBy, count, aliasTable, aliasCriterion); - System.out.println("[getHumanLabeledDocumentsByCrisisIDUserID] docList = " + docList); if (docList != null) { - System.out.println("Fetched size = " + docList.size()); Set docSet = new TreeSet(new DocumentComparator()); docSet.addAll(docList); - System.out.println("Sizeof document collection set = " + docSet.size()); // First get all labels for the fetched documents labeledDocList = new ArrayList(); @@ -1260,17 +1229,6 @@ public boolean deleteTask(Long crisisID, Long userID) { return success; } - /* - public static void main(String args[]) { - TaskManagerRemote tm = new TaskManagerBean(); - Map paramMap = new HashMap(); - paramMap.put("setHasHumanLabels", new Boolean(false).toString()); - paramMap.put("setCrisisID", new Long(117L).toString()); - qa.qcri.aidr.task.entities.Document newDoc = tm.setTaskParameter(qa.qcri.aidr.task.entities.Document.class, 4579275L, paramMap); - System.out.println("newDoc = " + newDoc.getDocumentID() + ": " + newDoc.isHasHumanLabels()); - } - */ - private class DocumentComparator implements Comparator { @Override From 3282770777aecd5047ec0a8531e8773bf25ab997 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Fri, 31 Jul 2015 17:24:38 +0530 Subject: [PATCH 29/44] Added logs in AIDR-Common. --- .../aidr/common/code/DateFormatConfig.java | 18 ++++++++++++--- .../aidr/common/code/FasterXmlWrapper.java | 6 ++++- .../qcri/aidr/common/code/JacksonWrapper.java | 5 ++-- .../filter/ClassifiedFilteredTweet.java | 2 +- .../common/filter/DeserializeFilters.java | 3 +-- .../common/filter/FilterQueryMatcher.java | 23 ++++++++----------- .../qa/qcri/aidr/common/util/EmailClient.java | 10 ++------ 7 files changed, 37 insertions(+), 30 deletions(-) diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/code/DateFormatConfig.java b/aidr-common/src/main/java/qa/qcri/aidr/common/code/DateFormatConfig.java index c877af5bb..f22b450f1 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/code/DateFormatConfig.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/code/DateFormatConfig.java @@ -30,6 +30,7 @@ public class DateFormatConfig { * @param timeString date as a string, conforming to oldFormat * @return a Hashmap containing and, * + * @throws ParseException */ public static Map convertBetweenDateFormats(final String oldFormat, final String newFormat, final String timeString) throws ParseException { Map convertedDate = new HashMap(2); @@ -38,7 +39,13 @@ public static Map convertBetweenDateFormats(final String oldForm newDateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); if (timeString != null) { SimpleDateFormat oldDateFormat = new SimpleDateFormat(oldFormat); - Date newDate = oldDateFormat.parse(timeString); + Date newDate = null; + try { + newDate = oldDateFormat.parse(timeString); + } catch (ParseException e) { + logger.error("Exception while parsing the date:"+timeString); + throw e; + } if (newDate != null) { convertedDate.put("date", newDate); convertedDate.put("dateString", newDateFormat.format(newDate)); @@ -63,10 +70,15 @@ public static Map convertBetweenDateFormats(final String oldForm * @return a Date object containing the date * @throws ParseException */ - public static Date getDateFromString(final String dateFormat, final String timeString) throws ParseException { + public static Date getDateFromString(final String dateFormat, final String timeString) throws ParseException { if (timeString != null) { SimpleDateFormat simpleDateFormat = new SimpleDateFormat(dateFormat); - return simpleDateFormat.parse(timeString); + try { + return simpleDateFormat.parse(timeString); + } catch (ParseException e) { + logger.error("Exception while parsing the date: "+timeString); + throw e; + } } return null; } diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/code/FasterXmlWrapper.java b/aidr-common/src/main/java/qa/qcri/aidr/common/code/FasterXmlWrapper.java index b6968a01d..1780a8560 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/code/FasterXmlWrapper.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/code/FasterXmlWrapper.java @@ -1,17 +1,21 @@ package qa.qcri.aidr.common.code; +import org.apache.log4j.Logger; + import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; public class FasterXmlWrapper { + private static Logger logger = Logger.getLogger(FasterXmlWrapper.class); + public static ObjectMapper getObjectMapper() { try { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); return objectMapper; } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception in FasterXmlWrapper " + e.getStackTrace()); return null; } } diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/code/JacksonWrapper.java b/aidr-common/src/main/java/qa/qcri/aidr/common/code/JacksonWrapper.java index 352756e6f..7cb5ecfb2 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/code/JacksonWrapper.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/code/JacksonWrapper.java @@ -1,5 +1,6 @@ package qa.qcri.aidr.common.code; +import org.apache.log4j.Logger; import org.codehaus.jackson.map.DeserializationConfig; import org.codehaus.jackson.map.ObjectMapper; @@ -8,7 +9,7 @@ * */ public class JacksonWrapper { - + private static Logger logger = Logger.getLogger(JacksonWrapper.class); /** * Generates an {@link ObjectMapper} and configures it so that it does not fail on unknown properties. * @@ -20,7 +21,7 @@ public static ObjectMapper getObjectMapper() { objectMapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); return objectMapper; } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception in JacksonWrapper "+e.getStackTrace()); return null; } } diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java index b4a97f66e..7f991e3d9 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/ClassifiedFilteredTweet.java @@ -154,7 +154,7 @@ public ClassifiedFilteredTweet deserialize(String rawJsonString) { } return null; } catch (Exception e) { - //logger.error("Exception in json parsing for string: " + rawJsonString); + logger.error("Exception in json parsing for string: " + rawJsonString, e); return null; } } diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/DeserializeFilters.java b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/DeserializeFilters.java index 22999d291..87b87a264 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/DeserializeFilters.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/DeserializeFilters.java @@ -36,8 +36,7 @@ public JsonQueryList deserializeConstraints(final String queryString) { GenericInputQuery constraint = jsonObject.fromJson(q, GenericInputQuery.class); queryList.createConstraint(constraint); } catch (Exception e) { - logger.error("Error in deserializing received constraints"); - logger.error(e); + logger.error("Error in deserializing received constraints",e); return null; } } diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/FilterQueryMatcher.java b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/FilterQueryMatcher.java index 9572fb6c1..7e6040ba3 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/filter/FilterQueryMatcher.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/filter/FilterQueryMatcher.java @@ -4,7 +4,6 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Date; import java.util.TimeZone; import org.apache.log4j.Logger; @@ -13,10 +12,10 @@ import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.annotate.JsonSerialize; -import com.google.gson.Gson; - import qa.qcri.aidr.common.code.DateFormatConfig; +import com.google.gson.Gson; + public class FilterQueryMatcher { private static Logger logger = Logger.getLogger(FilterQueryMatcher.class); @@ -41,7 +40,7 @@ public boolean isDateQuery(String queryString) { DateQueryJsonObject dateQuery = mapper.readValue(queryString, DateQueryJsonObject.class); return true; } catch (Exception e) { - logger.error("Exception", e); + logger.error("Exception while parsing the date query", e); } } return false; @@ -56,7 +55,7 @@ public boolean isClassifierQuery(String queryString) { return true; } catch (Exception e) { logger.debug("Exception in deserializing using Jackson readValue()"); - logger.error("Exception", e); + logger.error("Exception in deserializing using Jackson readValue()", e); } } return false; @@ -169,12 +168,11 @@ public QueryJsonObject serializeQuery(String queryString) { queryObject = mapper.readValue(queryString, DateQueryJsonObject.class); logger.info("DateQueryObject: " + queryObject.toString()); } catch (JsonParseException e) { - logger.error("JsonParseException for DateQueryJsonObject attempt"); + logger.error("JsonParseException for DateQueryJsonObject attempt",e); } catch (JsonMappingException e) { - logger.error("JsonMappingException for DateQueryJsonObject attempt"); + logger.error("JsonMappingException for DateQueryJsonObject attempt",e); } catch (IOException e) { - logger.error("IOException for DateQueryJsonObject attempt"); - logger.error("Exception", e); + logger.error("IOException for DateQueryJsonObject attempt",e); } } else if (isClassifierQuery(queryString)) { queryObject = new ClassifierQueryJsonObject(); @@ -188,8 +186,7 @@ public QueryJsonObject serializeQuery(String queryString) { } catch (JsonMappingException e) { logger.error("JsonMappingException for ClassifierQueryJsonObject attempt"); } catch (IOException e) { - logger.error("IOException for ClassifierQueryJsonObject attempt"); - logger.error("Exception", e); + logger.error("IOException for ClassifierQueryJsonObject attempt",e); } } return queryObject; @@ -227,7 +224,7 @@ public void buildMatcherArray(JsonQueryList qList) { } } - public static void main(String args[]) { + /*public static void main(String args[]) { System.out.println("In main - testing code"); ArrayList temp = new ArrayList(); @@ -271,5 +268,5 @@ public static void main(String args[]) { ++i; } System.out.println("[main] Result of matching: " + test.getMatcherResult(testTweet)); - } + }*/ } diff --git a/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java b/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java index 4662a0f7e..db57ec7ea 100644 --- a/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java +++ b/aidr-common/src/main/java/qa/qcri/aidr/common/util/EmailClient.java @@ -1,9 +1,6 @@ package qa.qcri.aidr.common.util; import java.io.UnsupportedEncodingException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; import java.util.Properties; import javax.mail.Message; @@ -52,9 +49,6 @@ protected PasswordAuthentication getPasswordAuthentication() { // Set Subject: header field message.setSubject(subject); - Date d = new Date(System.currentTimeMillis()); - String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); - // Now set the actual message message.setText(body); @@ -67,13 +61,13 @@ protected PasswordAuthentication getPasswordAuthentication() { Transport.send(message); logger.info("Sent message successfully to: " + recipients); } catch (AddressException e) { - e.printStackTrace(); + logger.error("Recipients address format exception"+e); } catch (MessagingException mex) { logger.error("Unable to send email to " + recipients + " due to " + mex.getCause()); mex.printStackTrace(); } catch (UnsupportedEncodingException e) { - e.printStackTrace(); + logger.error("Unsupported encoding scheme used"); } } } From 2b2080f6cb312283c74e27ecd3c23987636087a6 Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Fri, 31 Jul 2015 17:52:24 +0530 Subject: [PATCH 30/44] Commit for #97809356 : aidr-tagger logging --- .../java/qa/qcri/aidr/predict/DataStore.java | 9 +-- .../classification/LabelingTaskWriter.java | 1 + .../predict/classification/nominal/Model.java | 9 ++- .../nominal/ModelController.java | 8 +-- .../nominal/ModelRetrainTrigger.java | 1 - .../aidr/predict/common/TaggerErrorLog.java | 46 +++++++++++++++ .../common/TaskManagerEntityMapper.java | 59 +------------------ .../communication/HttpInputManager.java | 4 +- .../communication/HttpInputWorker.java | 5 +- .../communication/HttpOutputManager.java | 4 +- .../communication/HttpOutputWorker.java | 9 ++- .../predict/communication/OutputMatcher.java | 2 +- .../qa/qcri/aidr/predict/data/Document.java | 10 +--- .../predict/data/DocumentJSONConverter.java | 5 +- .../java/qa/qcri/aidr/predict/data/SMS.java | 1 - 15 files changed, 79 insertions(+), 94 deletions(-) create mode 100644 aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerErrorLog.java diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/DataStore.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/DataStore.java index 8b591d25b..4a886783d 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/DataStore.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/DataStore.java @@ -34,6 +34,7 @@ import qa.qcri.aidr.predict.common.Helpers; import qa.qcri.aidr.predict.common.TaggerConfigurationProperty; import qa.qcri.aidr.predict.common.TaggerConfigurator; +import qa.qcri.aidr.predict.common.TaggerErrorLog; import qa.qcri.aidr.predict.data.Document; import qa.qcri.aidr.predict.dbentities.ModelFamilyEC; import qa.qcri.aidr.predict.dbentities.NominalAttributeEC; @@ -83,7 +84,7 @@ public static synchronized void initializeJedisPool() throws Exception { jedisPool = new JedisPool(new JedisPoolConfig(), TaggerConfigurator .getInstance().getProperty( TaggerConfigurationProperty.REDIS_HOST)); - logger.info("Initialized jedisPool = " + jedisPool); + logger.info("Initialized jedisPool = " + jedisPool); } else { logger.warn("Attempting to initialize an active JedisPool!"); } @@ -95,12 +96,14 @@ public static void initDBPools() { } catch (Exception e1) { logger.error("Unable to allocate JEDIS Pool!"); logger.error("Exception", e1); + TaggerErrorLog.sendErrorMail("Redis", "Could not establish Redis connection. " + e1.getMessage()); } try { initializeMySqlPool(); } catch (Exception e) { logger.error("Unable to allocate MySQL Pool!"); logger.error("Exception", e); + TaggerErrorLog.sendErrorMail("Mysql connection", "Could not initialize mysql connection. " + e.getMessage()); } } @@ -127,7 +130,6 @@ public static void initTaskManager() { InitialContext ctx = new InitialContext(); taskManager = (TaskManagerRemote) ctx.lookup(DataStore.remoteEJBJNDIName); - System.out.println("taskManager: " + taskManager + ", time taken to initialize = " + (System.currentTimeMillis() - startTime)); logger.info("taskManager: " + taskManager + ", time taken to initialize = " + (System.currentTimeMillis() - startTime)); if (taskManager != null) { logger.info("Success in connecting to remote EJB to initialize taskManager"); @@ -168,10 +170,9 @@ public static Jedis getJedisConnection() { return jedisPool.getResource(); } } catch (Exception e) { - System.out - .println("Could not establish Redis connection. Is the Redis server running?"); logger.error("Could not establish Redis connection. Is the Redis server running?"); logger.error("Exception", e); + TaggerErrorLog.sendErrorMail("Redis", "Could not establish Redis connection. " + e.getMessage()); } return null; } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/LabelingTaskWriter.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/LabelingTaskWriter.java index 659cb34e8..bde520334 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/LabelingTaskWriter.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/LabelingTaskWriter.java @@ -117,6 +117,7 @@ void writeToDB() { try { Thread.sleep(200); } catch (InterruptedException e) { + logger.warn("Sleep thread interrupted."); } } } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/Model.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/Model.java index 411ccf362..873be8567 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/Model.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/Model.java @@ -61,7 +61,7 @@ public Evaluation evaluate(Instances evaluationSet) throws Exception { double recTemp = evaluation.recall(i); double aucTemp = evaluation.areaUnderROC(i); //.areaUnderPRC(i); if (!(aucTemp >= 0 && aucTemp <= 1)) { - logger.error("AUC is not available for the trained model"); + logger.warn("AUC is not available for the trained model"); aucTemp = 0; } @@ -104,8 +104,10 @@ public int getAttributeID() { } public NominalLabelBC classify(Document item) { - if (modelID == -1) - throw new RuntimeException("Model has not been initialized"); + if (modelID == -1) { + logger.error("Model has not been initialized"); + throw new RuntimeException("Model has not been initialized"); + } ArrayList wordSets = item.getFeatures(WordSet.class); if (wordSets.isEmpty()) { @@ -188,6 +190,7 @@ public void setTrainingSampleCount(int count) { public int getTrainingSampleCount() { if (trainingSampleCount == null) { + logger.error("trainingSampleCount has not been set"); throw new RuntimeException("trainingSampleCount has not been set"); } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelController.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelController.java index bb103d525..ba067dd1d 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelController.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelController.java @@ -164,7 +164,7 @@ private void onRetrainModel(CrisisAttributePair info) { } private void replaceModel(int eventID, int attributeID, Model newModel) { - logger.error("Replacing model for event " + eventID + logger.info("Replacing model for event " + eventID + " and attribute " + attributeID); // Insert the new model into the database and update the currentModelID @@ -172,6 +172,7 @@ private void replaceModel(int eventID, int attributeID, Model newModel) { int modelID = DataStore.saveModelToDatabase(eventID, attributeID, newModel); if (modelID == DataStore.MODEL_ID_ERROR) { + logger.error("Model was not correctly written to the database. Aborting write to disk."); throw new RuntimeException("Model was not correctly written to the database. Aborting write to disk."); } @@ -218,7 +219,7 @@ private boolean loadModel(int eventID, int attributeID, int modelID) { String path = getModelPath(eventID, attributeID, modelID); o = weka.core.SerializationHelper.readAll(path); } catch (Exception e) { - System.out.println("Could not load model from disk (crisis " + eventID + logger.error("Could not load model from disk (crisis " + eventID + ", attribute " + attributeID + ", model " + modelID + "). Delete model reference in DB and retrain? [y/n]"); try { @@ -227,9 +228,8 @@ private boolean loadModel(int eventID, int attributeID, int modelID) { onRetrainModel(new CrisisAttributePair(eventID, attributeID)); } } catch (IOException ex) { - //do nothing + logger.warn("Unable to read input."); } - //System.out.println(); return false; } Classifier classifier = (Classifier) o[0]; diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelRetrainTrigger.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelRetrainTrigger.java index eeb605504..171918424 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelRetrainTrigger.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelRetrainTrigger.java @@ -75,7 +75,6 @@ private int parseTrainingSamples() { && new Date().getTime() - consumptionStart < timeThreshold) { logger.info("A training sample has arrived"); - System.out.println("A training sample has arrived"); // Parse notification containing event id and attribute ids JSONObject obj = new JSONObject(line); diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerErrorLog.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerErrorLog.java new file mode 100644 index 000000000..64829d102 --- /dev/null +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaggerErrorLog.java @@ -0,0 +1,46 @@ +/** + * + */ +package qa.qcri.aidr.predict.common; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Form; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.log4j.Logger; +import org.glassfish.jersey.jackson.JacksonFeature; + +/** + * @author Latika + * TODO centralize in all modules + */ +public class TaggerErrorLog { + + private static Logger logger = Logger.getLogger(TaggerErrorLog.class); + + public static void sendErrorMail(String code, String errorMsg) { + Response clientResponse = null; + Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build(); + try { + WebTarget webResource = client.target(TaggerConfigurator.getInstance() + .getProperty(TaggerConfigurationProperty.TAGGER_API) + "/misc/sendErrorEmail"); + + Form form = new Form(); + form.param("module", "AIDRTagger"); + form.param("code", code); + form.param("description", errorMsg); + + clientResponse = webResource.request().post( + Entity.entity(form,MediaType.APPLICATION_FORM_URLENCODED),Response.class); + if (clientResponse.getStatus() != 200) { + logger.warn("Couldn't contact AIDRTaggerAPI for sending error message"); + } + } catch (Exception e) { + logger.error("Error in contacting AIDRTaggerAPI: " + clientResponse); + } + } +} diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaskManagerEntityMapper.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaskManagerEntityMapper.java index b11a487a1..36820c41a 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaskManagerEntityMapper.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/common/TaskManagerEntityMapper.java @@ -24,10 +24,7 @@ public E deSerializeList(String jsonString, TypeReference type) { return docList; } } catch (IOException e) { - logger.error("JSON deserialization exception"); - System.err.println("JSON deserialization exception"); - e.printStackTrace(); - + logger.error("JSON deserialization exception", e); } return null; } @@ -59,58 +56,4 @@ public String serializeTask(E task) { } return jsonString; } - - - - - - - -/* - public static void main(String args[]) { - TaskManagerEntityMapper mapper = new TaskManagerEntityMapper(); - qa.qcri.aidr.task.entities.Document doc = new qa.qcri.aidr.task.entities.Document(12345678L, false); - qa.qcri.aidr.task.entities.Document doc2 = new qa.qcri.aidr.task.entities.Document(12345679L, true); - - doc.setCrisisID(999L); - doc.setValueAsTrainingSample(0.8); - //doc.setSourceIP(77334455L); - - doc2.setCrisisID(1000L); - doc2.setValueAsTrainingSample(1.0); - //doc2.setSourceIP(555555L); - - String jsonString = mapper.serializeTask(doc); - TaggerDocument newDoc1 = mapper.transformDocument(doc); - System.out.println("Serialized json string: " + jsonString); - System.out.println("New document 1 = " + newDoc1.getDocumentID()); - - TaggerDocument newDoc2 = mapper.deSerialize(jsonString, TaggerDocument.class); - System.out.println("Deserialized document ID = " + newDoc2.getDocumentID()); - - List docList = new ArrayList(); - docList.add(doc); - docList.add(doc2); - String jsonString2 = mapper.serializeTask(docList); - System.out.println("Serialized List: " + jsonString2); - - List newDocList = mapper.deSerializeList(jsonString2, new TypeReference>() {}); - System.out.println("\nList deserialization: "); - for (TaggerDocument d: newDocList) { - System.out.println("Deserialized document ID = " + d.getDocumentID()); - } - - TaggerDocument doc3 = new TaggerDocument(1111111L, false); - doc3.setCrisisID(111L); - doc3.setValueAsTrainingSample(0.8); - //doc3.setSourceIP(77334455L); - - qa.qcri.aidr.task.entities.Document newDoc3 = mapper.reverseTransformDocument(doc3); - String jsonString3 = mapper.serializeTask(newDoc3); - System.out.println("Serialized TaggerDocument: " + jsonString3); - - qa.qcri.aidr.task.entities.Document newDoc4 = mapper.deSerialize(jsonString3, qa.qcri.aidr.task.entities.Document.class); - System.out.println("Deserialized document ID: " + newDoc4.getDocumentID()); - } -*/ } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpInputManager.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpInputManager.java index eeb3be61c..ca4720e5f 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpInputManager.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpInputManager.java @@ -50,7 +50,7 @@ public void run() { } } } catch (IOException e) { - logger.error("Could not listen on port " + logger.warn("Could not listen on port " + Integer.parseInt(TaggerConfigurator .getInstance().getProperty( TaggerConfigurationProperty.HTTP_INPUT_PORT))); @@ -68,7 +68,7 @@ protected void finalize() { try { server.close(); } catch (IOException e) { - logger.error("Could not close socket"); + logger.warn("Could not close socket"); } } } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpInputWorker.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpInputWorker.java index 529e74cee..60950adc2 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpInputWorker.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpInputWorker.java @@ -42,7 +42,7 @@ public void run() { in = new BufferedReader(new InputStreamReader( client.getInputStream())); } catch (IOException e) { - logger.error("Could not create input stream reader. " + e.getMessage()); + logger.warn("Could not create input stream reader. " + e.getMessage()); return; } logger.info("Created new InputWorker (" + connectionInstanceID @@ -67,7 +67,6 @@ public void run() { + connectionInstanceID); } - logger.info("Closing connection " + connectionInstanceID); try { client.close(); } catch (IOException e) { @@ -97,7 +96,7 @@ private void enqueue(Document doc) { TaggerConfigurationProperty.REDIS_FOR_EXTRACTION_QUEUE) .getBytes(), Serializer.serialize(doc)); } catch (IOException e) { - logger.error("Error when serializing input document : " + doc != null ? doc.toString() : "no doc"); + logger.warn("Error when serializing input document : " + doc != null ? doc.toString() : "no doc"); } finally { DataStore.close(jedis); } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpOutputManager.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpOutputManager.java index f8c6e3feb..abbc0d0cb 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpOutputManager.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpOutputManager.java @@ -52,7 +52,7 @@ public void run() { } } } catch (IOException e) { - logger.error("Could not listen on port " + logger.warn("Could not listen on port " + Integer .parseInt(TaggerConfigurator .getInstance() @@ -70,7 +70,7 @@ protected void finalize() { try { server.close(); } catch (IOException e) { - logger.error("Could not close socket", e); + logger.warn("Could not close socket", e); } } } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpOutputWorker.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpOutputWorker.java index bc630a4ec..06a1de3ed 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpOutputWorker.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/HttpOutputWorker.java @@ -123,10 +123,9 @@ OutputFilter getFilter(Socket s) { return OutputFilter.fromJson(input); } } catch (IOException e) { - logger.error("IOException while waiting for POST data"); + logger.warn("IOException while waiting for POST data"); } - logger.warn("Bad POST data."); return null; } } @@ -153,7 +152,7 @@ private HttpOutputWorker(Socket s, OutputFilter o) { try { clientStream = new PrintWriter(client.getOutputStream(), true); } catch (IOException e) { - logger.error("Could not get output stream for worker " + logger.warn("Could not get output stream for worker " + connectionInstanceID); } } @@ -165,7 +164,7 @@ public void push(String doc) { } if (clientStream.checkError()) { // Inform subscribers that the connection has closed - logger.info("Could not write to client, closing connection"); + logger.warn("Could not write to client, closing connection"); close(); } } @@ -177,7 +176,7 @@ public void close() { try { client.close(); } catch (IOException e) { - logger.error("IOException while closing client connection"); + logger.warn("IOException while closing client connection"); } if (!isClosed) onConnectionClosed.fire(this, null); diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/OutputMatcher.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/OutputMatcher.java index 7b33d8637..d41ab550e 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/OutputMatcher.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/communication/OutputMatcher.java @@ -81,7 +81,7 @@ private void pushToTaskWriteQueue(Document doc) { TaggerConfigurationProperty.REDIS_LABEL_TASK_WRITE_QUEUE) .getBytes(), Serializer.serialize(doc)); } catch (IOException e) { - logger.error("Exception while serializing DocumentSet."); + logger.warn("Exception while serializing DocumentSet."); } DataStore.close(jedis); } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/Document.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/Document.java index d1b098a5a..8ca1f0408 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/Document.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/Document.java @@ -1,21 +1,13 @@ package qa.qcri.aidr.predict.data; -import java.net.InetAddress; import java.util.ArrayList; -import java.util.List; -import javax.xml.bind.annotation.XmlTransient; - -import org.json.*; - -import com.fasterxml.jackson.annotation.JsonIgnore; +import org.json.JSONObject; import qa.qcri.aidr.predict.classification.DocumentLabel; import qa.qcri.aidr.predict.classification.DocumentLabelFilter; -import qa.qcri.aidr.predict.classification.nominal.NominalLabelBC; import qa.qcri.aidr.predict.common.DocumentType; import qa.qcri.aidr.predict.common.Helpers; -import qa.qcri.aidr.predict.dbentities.NominalLabel; import qa.qcri.aidr.predict.dbentities.TaggerDocument; import qa.qcri.aidr.predict.featureextraction.DocumentFeature; import qa.qcri.aidr.predict.featureextraction.FeatureExtractor; diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/DocumentJSONConverter.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/DocumentJSONConverter.java index 199c9ddec..c0699ddbc 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/DocumentJSONConverter.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/DocumentJSONConverter.java @@ -306,9 +306,12 @@ private static ModelFamilyEC getModelFamily(int crisisID, String attributeCode) } if (!activeModelFamiliesByCode.containsKey(crisisID) - || !activeModelFamiliesByCode.get(crisisID).containsKey(attributeCode)) + || !activeModelFamiliesByCode.get(crisisID).containsKey(attributeCode)) { + + logger.error("ModelInfo is missing for crisis " + crisisID + " and attribute " + attributeCode); throw new RuntimeException( "ModelInfo is missing for crisis " + crisisID + " and attribute " + attributeCode); + } return activeModelFamiliesByCode.get(crisisID).get(attributeCode); } diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/SMS.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/SMS.java index 972c7150f..310080807 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/SMS.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/data/SMS.java @@ -2,7 +2,6 @@ import java.io.Serializable; -import qa.qcri.aidr.predict.classification.geo.GeoLabel; import qa.qcri.aidr.predict.common.DocumentType; /** From c643e0603e8340e85e655c58e6ed99995d2baae3 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Sat, 1 Aug 2015 13:47:23 +0530 Subject: [PATCH 31/44] Added logs in Aidr-Manager. --- .../manager/controller/BaseController.java | 2 - .../controller/CollectionController.java | 43 +-- .../manager/controller/EmscController.java | 25 +- .../manager/controller/PublicController.java | 29 +- .../manager/controller/TaggerController.java | 115 ++++--- .../controller/UITemplateController.java | 28 +- .../manager/controller/UserController.java | 2 +- .../aidr/manager/dto/TaggerCrisisRequest.java | 4 - .../qa/qcri/aidr/manager/dto/TaggerModel.java | 1 - .../qa/qcri/aidr/manager/dto/TaskAnswer.java | 1 - .../AidrAuthenticateToken.java | 12 +- .../impl/CollectionRepositoryImpl.java | 6 +- .../repository/impl/UserRepositoryImpl.java | 5 +- .../AjaxTimeoutRedirectFilter.java | 1 + .../impl/CollectionLogServiceImpl.java | 30 +- .../service/impl/CollectionServiceImpl.java | 43 ++- .../service/impl/TaggerServiceImpl.java | 292 ++++++++---------- .../service/impl/UITemplateServiceImpl.java | 21 +- .../SpringSocialAuthenticationFilter.java | 1 + .../SpringSocialUserDetailService.java | 3 + .../aidr/manager/util/JsonDataValidator.java | 6 +- .../manager/util/JsonDateDeSerializer.java | 27 +- .../aidr/manager/util/JsonDateSerializer.java | 23 +- .../qcri/aidr/manager/util/ScheduledTask.java | 12 +- 24 files changed, 367 insertions(+), 365 deletions(-) diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/BaseController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/BaseController.java index 0a854a12d..bd1c2b75f 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/BaseController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/BaseController.java @@ -5,9 +5,7 @@ import java.util.Map; import javax.annotation.Resource; -import javax.servlet.http.HttpSession; -import org.eclipse.persistence.sessions.Session; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/CollectionController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/CollectionController.java index d51b547be..1e0d10ace 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/CollectionController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/CollectionController.java @@ -28,6 +28,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; import java.util.Date; @@ -126,7 +127,8 @@ public Map updatePubliclyListed(AidrCollection collection) throws logger.info("[updatePubliclyListed] Save AidrCollection to Database having code : "+collection.getCode()); try{ AidrCollection dbCollection = collectionService.findById(collection.getId()); - System.out.println("[updatePubliclyListed] old status: " + dbCollection.getPubliclyListed() + ", new status: " + collection.getPubliclyListed()); + //System.out.println("[updatePubliclyListed] old status: " + dbCollection.getPubliclyListed() + ", new status: " + collection.getPubliclyListed()); + logger.info("[updatePubliclyListed] old status: " + dbCollection.getPubliclyListed() + ", new status: " + collection.getPubliclyListed()); dbCollection.setPubliclyListed(collection.getPubliclyListed()); collectionService.update(dbCollection); return getUIWrapper(true); @@ -171,7 +173,7 @@ public Map trash(@RequestParam Integer id) throws Exception { collection.getStatus().equals(CollectionStatus.RUNNING) || collection.getStatus().equals(CollectionStatus.RUNNING_WARNING)) { String msg = "Attempting to trash a running collection. Collection must be in stopped state!"; - logger.error(msg); + logger.warn(msg); return getUIWrapper(false, msg); } else { // Trash collection @@ -238,7 +240,7 @@ public Map untrash( AidrCollection collection) throws Exception { } }catch(Exception e){ String msg = "Error while untrashing AIDR Collection " + trashedCollection.getCode(); - logger.error(msg); + logger.error(msg,e); return getUIWrapper(false, msg); } } else { @@ -328,8 +330,7 @@ public Map update(AidrCollection collection) throws Exception { logger.error("Error in updating the crisis name from " + crisis.getName() + " to new name: " + collection.getName()); } } catch (AidrException e) { - logger.error("Error in updating the crisis name from " + dbCollection.getName() + " to new name: " + collection.getName()); - logger.error("exception", e); + logger.error("Error in updating the crisis name from " + dbCollection.getName() + " to new name: " + collection.getName(), e); } } // if collection type was changed and if crisis for this collection exists so we need to update crisis type @@ -345,15 +346,14 @@ public Map update(AidrCollection collection) throws Exception { logger.error("Error in updating the crisis type from " + crisis.getCrisisType().getCrisisTypeID() + " to new name: " + collection.getCrisisType()); } } catch (AidrException e) { - logger.error("Error in updating the crisis type from " + dbCollection.getCrisisType() + " to new type: " + collection.getCrisisType()); - logger.error("exception", e); + logger.error("Error in updating the crisis type from " + dbCollection.getCrisisType() + " to new type: " + collection.getCrisisType(), e); } } } return getUIWrapper(true); }catch(Exception e){ - logger.error("Error while Updating AidrCollection Info into database", e); + logger.error("Error while Updating AidrCollection Info into database for collection: "+collection.getCode(), e); return getUIWrapper(false); } } @@ -383,8 +383,10 @@ public Map findAll(@RequestParam Integer start, @RequestParam In @DefaultValue("no") @QueryParam("trashed") String trashed) throws Exception { start = (start != null) ? start : 0; limit = (limit != null) ? limit : 50; + String userName=""; UserEntity userEntity = getAuthenticatedUser(); if (userEntity != null) { + userName = userEntity.getUserName(); List dtoList = new ArrayList(); Integer count = 0; boolean onlyTrashed = false; @@ -410,7 +412,7 @@ public Map findAll(@RequestParam Integer start, @RequestParam In } } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while finding all collections for current user: "+userName, e); } return getUIWrapper(dtoList, count.longValue()); } @@ -466,7 +468,7 @@ public Map start(@RequestParam Integer id) throws Exception { return getUIWrapper(dto, true); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while starting the collection by Id: "+id); } return getUIWrapper(false); } @@ -507,7 +509,7 @@ public Map refreshCount(@RequestParam Integer id) throws Exceptio return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while refreshing the count for collectionId: "+id); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } return getUIWrapper(dto,true); @@ -540,8 +542,7 @@ public Map generateCSVLink(@RequestParam String code) throws Exce return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - logger.error("Exception in generating CSV download link for collection: " + code); - logger.error("exception", e); + logger.error("Exception in generating CSV download link for collection: " + code, e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -558,7 +559,7 @@ public Map generateTweetIdsLink(@RequestParam String code) throws return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - logger.error("Exception in generating CSV TweetIDs download link for collection: " + code); + logger.error("Exception in generating CSV TweetIDs download link for collection: " + code, e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -592,14 +593,14 @@ public Map getAllRunning(@RequestParam Integer start, try { taggersForCollections = taggerService.getTaggersForCollections(collectionCodes); } catch (Exception e) { - e.printStackTrace(); + logger.error("[getAllRunning.action] Error while getting taggers for collections: "+Arrays.toString(collectionCodes.toArray()), e); } Map totalCountsFromLogForCollections = Collections.emptyMap(); try { totalCountsFromLogForCollections = collectionLogService.countTotalDownloadedItemsForCollectionIds(collectionIds); } catch (Exception e) { - e.printStackTrace(); + logger.error("[getAllRunning.action] Error while getting total counts from log for collectionIds: "+Arrays.toString(collectionIds.toArray()), e); } for (AidrCollection collection : collections) { @@ -658,14 +659,14 @@ public Map getAllStopped(@RequestParam Integer start, try { taggersForCollections = taggerService.getTaggersForCollections(collectionCodes); } catch (Exception e) { - e.printStackTrace(); + logger.error("[getAllStopped.action] Error while getting taggers for collections: "+Arrays.toString(collectionCodes.toArray()), e); } Map totalCountsFromLogForCollections = Collections.emptyMap(); try { totalCountsFromLogForCollections = collectionLogService.countTotalDownloadedItemsForCollectionIds(collectionIds); } catch (Exception e) { - e.printStackTrace(); + logger.error("[getAllStopped.action] Error while getting total counts from log for collectionIds: "+Arrays.toString(collectionIds.toArray()), e); } for (AidrCollection collection : collections) { @@ -754,7 +755,7 @@ public Map generateJSONLink(@RequestParam String code, return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating JSON Link for collection: "+code +"and jsonType: "+jsonType, e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -772,7 +773,7 @@ public Map generateJsonTweetIdsLink(@RequestParam String code, return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating JSON Tweet Ids Link for collection: "+code +"and jsonType: "+jsonType, e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -798,7 +799,7 @@ public Map getTwitterUserID(@RequestParam(value ="userId", requi return getUIWrapper(null, false, 0L, "User ID provided is incorrect or doesn't exist"); } } catch (Exception e) { - logger.error("exception", e); + logger.error("Error while getting twitter userIds", e); return getUIWrapper(false, "Exception in twitter user data lookup."); } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/EmscController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/EmscController.java index a28e561d6..723443d61 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/EmscController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/EmscController.java @@ -1,5 +1,15 @@ package qa.qcri.aidr.manager.controller; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.Consumes; +import javax.ws.rs.core.MediaType; + import org.apache.log4j.Logger; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -8,19 +18,17 @@ import org.springframework.beans.propertyeditors.CustomDateEditor; import org.springframework.stereotype.Controller; import org.springframework.web.bind.WebDataBinder; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; + import qa.qcri.aidr.manager.hibernateEntities.AidrCollection; import qa.qcri.aidr.manager.hibernateEntities.UserEntity; import qa.qcri.aidr.manager.service.CollectionService; import qa.qcri.aidr.manager.util.CollectionStatus; import qa.qcri.aidr.manager.util.JsonDataValidator; - -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.core.MediaType; -import java.text.SimpleDateFormat; -import java.util.*; /** * @deprecated replaced by {@link @PublicController} */ @@ -44,7 +52,6 @@ public void initBinder(WebDataBinder binder) { @Consumes(MediaType.APPLICATION_JSON) @ResponseBody public Map saveStart(@RequestBody final String jsonString) throws Exception { - // // logger.info("saveStart CeaController : "+jsonString); try{ if(jsonString == null){ diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/PublicController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/PublicController.java index 2a0463116..e2305b284 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/PublicController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/PublicController.java @@ -74,12 +74,9 @@ public Map update(@RequestBody final String jsonCollection ) thro return getUIWrapper(false); } - JSONParser parser = new JSONParser(); Object obj = parser.parse(jsonCollection); - - JSONObject jsonObject = (JSONObject) obj; String geoString = (String)jsonObject.get("geo"); @@ -130,7 +127,7 @@ public Map update(@RequestBody final String jsonCollection ) thro } } - // save current state of the collection to collectionLog + // save current state of the collection to collectionLog AidrCollectionLog collectionLog = new AidrCollectionLog(dbCollection); collectionLog.setEndDate(collectionLogEndData); collectionLogService.create(collectionLog); @@ -151,7 +148,7 @@ public Map update(@RequestBody final String jsonCollection ) thro return getUIWrapper(true); }catch(Exception e){ - logger.error(String.format("Exception while Updating AidrCollection : %s", e)); + logger.error(String.format("Exception while Updating AidrCollection : "+jsonCollection, e)); return getUIWrapper(false); } } @@ -160,10 +157,12 @@ public Map update(@RequestBody final String jsonCollection ) thro @ResponseBody public Map findByRequestCode(@QueryParam("code") String code) throws Exception { try { + logger.info("Finding collection by code: "+code); AidrCollection data = collectionService.findByCode(code); return getUIWrapper(data, true); } catch (Exception e) { + logger.error("Exception while finding collection by code: "+code, e); return getUIWrapper(false); } @@ -183,7 +182,7 @@ public Map findAll(@RequestParam Integer start, @RequestParam In return getUIWrapper(data, true); } catch (Exception e) { - logger.error("Error in findAll."); + logger.error("Error in find All collection for public",e); return getUIWrapper(false); } @@ -218,7 +217,7 @@ public Map findAllRunning(@RequestParam Integer start, @RequestP return getUIWrapper(dtoList, count.longValue()); } catch (Exception e) { - logger.error("Error in findAllRunning."); + logger.error("Error in find All Running collection for public",e); return getUIWrapper(false); } @@ -248,7 +247,7 @@ public Map findAllRunningWithNoOutput(@RequestParam Integer star return getUIWrapper(dtoList, count.longValue()); } catch (Exception e) { - logger.error("Error in findAllRunningWithNoOutput."); + logger.error("Error in find All Running collection With No Output for public",e); return getUIWrapper(false); } @@ -317,7 +316,7 @@ public Map generateTweetIdsLink(@RequestParam String code) throws return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } catch (Exception e) { - logger.error("Error in generateTweetIdsLink for codd : " + code); + logger.error("Error in generateTweetIdsLink for collection : " + code, e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -329,7 +328,7 @@ public Map getAttributesAndLabelsByCrisisId(@RequestParam Integer try { result = taggerService.getAttributesAndLabelsByCrisisId(id); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while getting attributes and labels for crisis: " + id, e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } return getUIWrapper(result,true); @@ -343,7 +342,7 @@ public Map loadLatestTweets(@RequestParam String code, @RequestPa try { result = taggerService.loadLatestTweets(code, constraints); } catch (Exception e) { - logger.error("Error in loadLatestTweets for code : " + code + " and constraints : " + constraints); + logger.error("Error while loading latest tweets for collection : " + code + " and constraints : " + constraints, e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } return getUIWrapper(result,true); @@ -368,7 +367,7 @@ public Map getPublicFlagStatus() { return runningCollections; } } catch (Exception e) { - logger.error("Unable to fetch list of running collections from DB"); + logger.error("Unable to fetch list of running collections from DB for public", e); } return null; } @@ -390,7 +389,7 @@ public Map getCollectionPublicFlagStatus(@QueryParam("channelCo return result; } } catch (Exception e) { - logger.error("Unable to fetch list of running collections from DB"); + logger.error("Unable to fetch running status for collection: "+channelCode,e); } return null; } @@ -413,7 +412,7 @@ public Map findTotalCount(final String collectionCode) throws E result.put(collectionCode, dto.getTotalCount()); return result; } catch (Exception e) { - logger.error("Unable to fetch total count of downloaded documents for collection = " + collectionCode); + logger.error("Unable to fetch total count of downloaded documents for collection = " + collectionCode, e); } return null; } @@ -481,7 +480,7 @@ private String getCrisisTypeName(int typeID){ } } catch (AidrException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + logger.error("Error while fetching all crisisTypes for public",e); } return name; diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java index f002c5e7a..c42713d55 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java @@ -78,7 +78,7 @@ public Map getAllCrisis() { try { return getUIWrapper(taggerService.getAllCrisisTypes(), true); } catch (AidrException e) { - logger.error(e.getMessage(), e); + logger.error("Error while fetching all crisisTypes", e); return getUIWrapper(false, e.getMessage()); } } @@ -100,7 +100,7 @@ public Map getCrisesByUserId() { return getUIWrapper(false, "Error while getting all crisis for user in Tagger"); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while getting all crisis for current user", e); return getUIWrapper(false, e.getMessage()); } } @@ -133,7 +133,7 @@ public Map createCrises(CrisisRequest crisisRequest) { return getUIWrapper(false, "Unable to create new user in predict DB"); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while creating new crisis", e); return getUIWrapper(false, e.getMessage()); } } @@ -147,7 +147,7 @@ public Map getAttributesForCrises(Integer id) { Integer taggerUserId = taggerService.isUserExistsByUsername(userName); return getUIWrapper(taggerService.getAttributesForCrises(id, taggerUserId), true); } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while getting attributes for crisisId:"+id, e); return getUIWrapper(false, e.getMessage()); } } @@ -171,7 +171,7 @@ public Map addAttributeToCrisis(Integer crisesId, Integer attrib return getUIWrapper("Error while adding attribute to crises", false); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while adding attributes to crisisId: "+crisesId, e); return getUIWrapper(false, e.getMessage()); } } @@ -191,7 +191,7 @@ public Map update(UpdateCrisisDTO dto) throws Exception { return getUIWrapper(updatedCrisis != null); }catch(Exception e){ - logger.error("Error while Updating Crisis in Tagger", e); + logger.error("Error while Updating CrisisCode: "+dto.getCode(), e); return getUIWrapper(false); } } @@ -214,7 +214,7 @@ public Map getModelsForCrisis(Integer id) { logger.info("cast long value: " + value); result.get(i).setTrainingExamples(value.longValue()); } catch (Exception e) { - logger.error("Error in getModelsForCrisis for id : " + id); + logger.error("Error in getModelsForCrisis for crisisId : " + id, e); } } logger.info("For model family id: " + result.get(i).getModelFamilyID() + ", set human labeled count = " + result.get(i).getTrainingExamples()); @@ -235,7 +235,7 @@ public Map getRetrainThreshold() { try { return getUIWrapper(taggerService.getRetainingThreshold(), true); } catch (AidrException e) { - logger.error(e.getMessage(), e); + logger.error("Error while getting retrained threshold", e); return getUIWrapper(false, e.getMessage()); } } @@ -247,7 +247,7 @@ public Map getAllLabelsForModel(Integer id, String code) { try { return getUIWrapper(taggerService.getAllLabelsForModel(id, code), true); } catch (AidrException e) { - logger.error(e.getMessage(), e); + logger.error("Error while getting All Labels For Model = " + id + ", and crisis = " + code, e); return getUIWrapper(false, e.getMessage()); } } @@ -269,7 +269,7 @@ public Map createAttribute(TaggerAttribute attribute) { return getUIWrapper(false, "Error while creating new attribute in Tagger"); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while creating a new attribute", e); return getUIWrapper(false, e.getMessage()); } } @@ -286,7 +286,7 @@ public Map getAttributeInfo(@RequestParam Integer id) { return getUIWrapper(false, "Error while getting attribute from Tagger"); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while getting attribute info for attributeId:"+id, e); return getUIWrapper(false, e.getMessage()); } } @@ -303,8 +303,8 @@ public Map deleteAttribute(@RequestParam Integer id) { return getUIWrapper(false, "Error while deleting attribute in Tagger"); } } catch (Exception e) { - logger.error(e.getMessage(), e); - return getUIWrapper(false, e.getMessage()); + logger.error("Error while deleting attributeId: "+id, e); + return getUIWrapper(false, e.getMessage()); } } @@ -320,7 +320,7 @@ public Map removeAttributeFromCrises(@RequestParam Integer id) { return getUIWrapper(false, "Error while remove classifier from crises in Tagger"); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while removing classifier from crises by modelFamilyID: "+id, e); return getUIWrapper(false, e.getMessage()); } } @@ -341,7 +341,7 @@ public Map updateAttribute(@RequestParam Integer attributeID, @Re } return getUIWrapper(updatedAttribute != null); }catch(Exception e){ - logger.error("Error while updating attribute in Tagger", e); + logger.error("Updating Attribute in Tagger having id " + attributeID, e); return getUIWrapper(false); } } @@ -369,7 +369,7 @@ public Map updateLabel(@RequestParam Integer labelID, } return getUIWrapper(updatedLabel != null); }catch(Exception e){ - logger.error("Error while updating label in Tagger", e); + logger.error("Error while updating label having id: "+labelID, e); return getUIWrapper(false); } } @@ -383,10 +383,10 @@ public Map createLabel(TaggerLabelRequest labelRequest) { if (response != null){ return getUIWrapper(response, true); } else { - return getUIWrapper(false, "Error while creating new label in Tagger"); + return getUIWrapper(false, "Error while creating a new label in Tagger"); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while creating a new label", e); return getUIWrapper(false, e.getMessage()); } } @@ -422,7 +422,7 @@ public Map getTrainingDataByModelIdAndCrisisId(@RequestParam Int response = taggerService.getTrainingDataByModelIdAndCrisisId(modelFamilyId, crisisId, start, limit, sortColumn, sortDirection); logger.info("For crisis ID " + crisisId + ", model family ID " + modelFamilyId + ", Returned response: " + response); } catch (AidrException e) { - logger.error(e.getMessage(), e); + logger.error("Error while Getting training data for CrisisID: "+crisisId+ " and ModelFamilyID: "+modelFamilyId, e); return getUIWrapper(false, e.getMessage()); } Integer total = 0; @@ -465,7 +465,7 @@ public Map crisisExists(@RequestParam String code) throws Excepti try { taggerCrisisExist = taggerService.isCrisesExist(code); } catch (AidrException e) { - e.printStackTrace(); + logger.error("Error while checking if crisis exist for the code: "+code +"\t"+e.getStackTrace()); return getUIWrapper(false); } if (taggerCrisisExist != null && taggerCrisisExist.getCrisisId() != null && taggerCrisisExist.getCrisisId() != 0){ @@ -489,7 +489,7 @@ public Map getAssignableTask(@RequestParam Integer id) throws Exc logger.info("sVar : " + sVar); return getUIWrapper(sVar, true); } catch (AidrException e) { - e.printStackTrace(); + logger.error("Error while getting assignable task for crisisID: "+id +"\t"+e.getStackTrace()); return getUIWrapper(e.getMessage(), false); } } @@ -506,7 +506,7 @@ public Map skipTask(@RequestParam Integer id) throws Exception { logger.info("sVar : " + sVar); return getUIWrapper(sVar, true); } catch (AidrException e) { - e.printStackTrace(); + logger.error("Error ehile skipping task for documentID: "+id +"\t"+e.getStackTrace()); return getUIWrapper(e.getMessage(), false); } } @@ -522,7 +522,7 @@ public Map getTemplateStatus(@RequestParam String code) throws Ex logger.info("sVar : " + sVar); return getUIWrapper(sVar, true); } catch (AidrException e) { - e.printStackTrace(); + logger.error("Error while getting template status for crisis with code: " + code +"\t"+e.getStackTrace()); return getUIWrapper(e.getMessage(), false); } } @@ -543,7 +543,7 @@ public Map saveTaskAnswer(TaskAnswerRequest taskAnswerRequest) { boolean result = taggerService.saveTaskAnswer(taskAnswer); return getUIWrapper(result); } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while saving TaskAnswer in AIDRCrowdsourcing", e); return getUIWrapper(false, e.getMessage()); } } @@ -581,7 +581,7 @@ public Map getModelHistoryByModelFamilyID(@RequestParam Integer return getUIWrapper(Collections.emptyList(), 0L); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while getting history records for Model by ModelFamilyId: "+id,e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -598,7 +598,7 @@ public Map deleteTrainingExample(@RequestParam Integer id) { return getUIWrapper(false, "Error while deleting Training Example in Tagger"); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while deleting Training Example by ID:"+id, e); return getUIWrapper(false, e.getMessage()); } } @@ -606,6 +606,7 @@ public Map deleteTrainingExample(@RequestParam Integer id) { @RequestMapping(value = "/pingService.action", method = RequestMethod.GET) @ResponseBody public Map pingService(@RequestParam String service) throws Exception { + logger.info("In pinging service of every module"); boolean result = false; try { if ("tagger".equals(service)) { @@ -621,7 +622,7 @@ public Map pingService(@RequestParam String service) throws Excep } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while pinging all modules",e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } return getUIWrapper(result, true); @@ -630,11 +631,12 @@ public Map pingService(@RequestParam String service) throws Excep @RequestMapping(value = "/getAttributesAndLabelsByCrisisId.action", method = RequestMethod.GET) @ResponseBody public Map getAttributesAndLabelsByCrisisId(@RequestParam Integer id) throws Exception { + logger.info("Getting attributes and labels by crisisID: "+id); String result = ""; try { result = taggerService.getAttributesAndLabelsByCrisisId(id); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while getting attributes and labels by crisisID: "+id, e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } return getUIWrapper(result,true); @@ -644,6 +646,7 @@ public Map getAttributesAndLabelsByCrisisId(@RequestParam Integer @RequestMapping(value = "/taggerGenerateCSVLink.action", method = RequestMethod.GET) @ResponseBody public Map generateCSVLink(@RequestParam String code) throws Exception { + logger.info("Received request for generating csv link for Collection: " + code); Map result = null; try { result = taggerService.generateCSVLink(code); @@ -653,7 +656,7 @@ public Map generateCSVLink(@RequestParam String code) throws Exce return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating csv link for colection: "+code +"/t"+e.getStackTrace()); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -662,7 +665,8 @@ public Map generateCSVLink(@RequestParam String code) throws Exce @RequestMapping(value = "/taggerGenerateTweetIdsLink.action", method = RequestMethod.GET) @ResponseBody public Map generateTweetIdsLink(@RequestParam String code) throws Exception { - System.out.println("[Controller generateTweetIdsLink] Received request for code: " + code); + //System.out.println("[Controller generateTweetIdsLink] Received request for code: " + code); + logger.info("Received request for generating tweetIds link for Collection: " + code); Map result = null; try { result = taggerService.generateTweetIdsLink(code); @@ -673,7 +677,7 @@ public Map generateTweetIdsLink(@RequestParam String code) throws return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating tweetIds link for colection: "+code +"/t"+e.getStackTrace()); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } //System.out.println("[Controller generateTweetIdsLink] Returning success: " + result); @@ -733,6 +737,7 @@ private List transformTaskAnswerRequestToTaskAnswer (TaskAnswerReque @ResponseBody public Map generateJSONLink(@RequestParam String code, @DefaultValue(DownloadType.TEXT_JSON) @QueryParam("jsonType") String jsonType) throws Exception { + logger.info("Received request for generating JSON link for Collection: " + code); Map result = null; try { result = taggerService.generateJSONLink(code, jsonType); @@ -742,7 +747,7 @@ public Map generateJSONLink(@RequestParam String code, return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating JSON link for colection: "+code +" & jsonType: "+jsonType+"/t"+e.getStackTrace()); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } //return getUIWrapper(result,true); @@ -753,7 +758,8 @@ public Map generateJSONLink(@RequestParam String code, @ResponseBody public Map generateJsonTweetIdsLink(@RequestParam String code, @DefaultValue(DownloadType.TEXT_JSON) @QueryParam("jsonType") String jsonType) throws Exception { - System.out.println("[Controller generateTweetIdsLink] Received request for code: " + code); + //System.out.println("[Controller generateTweetIdsLink] Received request for code: " + code); + logger.info("Received request for generating JSON TweetIds link for Collection: " + code); Map result = null; try { result = taggerService.generateJsonTweetIdsLink(code, jsonType); @@ -764,7 +770,7 @@ public Map generateJsonTweetIdsLink(@RequestParam String code, return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating JSON TweetIds link for colection: "+code +" & jsonType: "+jsonType+"/t"+e.getStackTrace()); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } //System.out.println("[Controller generateTweetIdsLink] Returning success: " + result); @@ -777,6 +783,7 @@ public Map generateJsonTweetIdsLink(@RequestParam String code, @ResponseBody public Map generateCSVFilteredLink(@RequestParam String code, String queryString) throws Exception { + logger.info("Received request for generating CSV filtered link for Collection: " + code); Map result = null; try { String userName = getAuthenticatedUserName(); @@ -789,7 +796,7 @@ public Map generateCSVFilteredLink(@RequestParam String code, return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating CSV filtered link for colection: "+code +"/t"+e.getStackTrace()); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -799,7 +806,8 @@ public Map generateCSVFilteredLink(@RequestParam String code, @ResponseBody public Map generateTweetIdsFilteredLink(@RequestParam String code, String queryString) throws Exception { - System.out.println("[Controller generateTweetIdsLink] Received request for code: " + code); + //System.out.println("[Controller generateTweetIdsLink] Received request for code: " + code); + logger.info("Received request for generating tweetIDs filtered link for Collection: " + code); Map result = null; try { String userName = getAuthenticatedUserName(); @@ -813,7 +821,7 @@ public Map generateTweetIdsFilteredLink(@RequestParam String code return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating tweetIDs filtered link for colection: "+code +"/t"+e.getStackTrace()); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } //System.out.println("[Controller generateTweetIdsLink] Returning success: " + result); @@ -826,6 +834,7 @@ public Map generateTweetIdsFilteredLink(@RequestParam String code public Map generateJSONFilteredLink(@RequestParam String code, String queryString, @DefaultValue(DownloadType.TEXT_JSON) @QueryParam("jsonType") String jsonType) throws Exception { + logger.info("Received request for generating JSON filtered link for Collection: " + code); Map result = null; try { String userName = getAuthenticatedUserName(); @@ -838,7 +847,7 @@ public Map generateJSONFilteredLink(@RequestParam String code, return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating JSON filtered link for colection: "+code +"/t"+e.getStackTrace()); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } //return getUIWrapper(result,true); @@ -850,7 +859,8 @@ public Map generateJSONFilteredLink(@RequestParam String code, public Map generateJsonTweetIdsFilteredLink(@RequestParam String code, String queryString, @DefaultValue(DownloadType.TEXT_JSON) @QueryParam("jsonType") String jsonType) throws Exception { - System.out.println("[Controller generateTweetIdsLink] Received request for code: " + code); + //System.out.println("[Controller generateTweetIdsLink] Received request for code: " + code); + logger.info("Received request for generating JSON TweetIds filtered link for Collection: " + code); Map result = null; try { String userName = getAuthenticatedUserName(); @@ -858,13 +868,13 @@ public Map generateJsonTweetIdsFilteredLink(@RequestParam String result = taggerService.generateJsonTweetIdsFilteredLink(code, queryString, jsonType, userName); if (result != null && result.get("url") != null) { - System.out.println("Returning success fo collection: " + code + ", response: " + result); + //System.out.println("Returning success fo collection: " + code + ", response: " + result); return getUIWrapper(result.get("url"),true, null, (String)result.get("message")); } else { return getUIWrapper(false, "Something wrong - no file generated!"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while generating JSON TweetIds filtered link for colection: "+code +" & jsonType: "+jsonType+"/t"+e.getStackTrace()); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } @@ -872,7 +882,8 @@ public Map generateJsonTweetIdsFilteredLink(@RequestParam String @RequestMapping(value = "/updateMobilePush.action", method = { RequestMethod.POST ,RequestMethod.GET }) @ResponseBody public Map updateMobilePushStatus(AidrCollection collection) throws Exception { - System.out.println("[updateMobilePushStatus: " + collection.getCode()); + //System.out.println("[updateMobilePushStatus: " + collection.getCode()); + logger.info("[updateMobilePushStatus: " + collection.getCode()); String result = ""; try { TaggerCrisis tagCrisis = taggerService.getCrisesByCode(collection.getCode()); @@ -880,7 +891,7 @@ public Map updateMobilePushStatus(AidrCollection collection) thro /// all clientapp based on crisisID should be enable to push to mobile app. } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while updating mobile push status for colection: "+collection.getCode() +"/t"+e.getStackTrace()); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } @@ -891,10 +902,11 @@ public Map updateMobilePushStatus(AidrCollection collection) thro @ResponseBody public Map getHumanLabeledDocumentsByCrisisID(Long crisisID, Integer count) throws Exception { try { + logger.info("Getting human labelled documents by crisisID: "+crisisID); TaggerResponseWrapper labeledDataList = taggerService.getHumanLabeledDocumentsByCrisisID(crisisID, count); return getUIWrapper(labeledDataList, true); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while getting human labelled documents by crisisID: "+crisisID +"/t"+e.getStackTrace()); return getUIWrapper(false, "Error in fetching human labeled documents"); } } @@ -903,10 +915,11 @@ public Map getHumanLabeledDocumentsByCrisisID(Long crisisID, Inte @ResponseBody public Map getHumanLabeledDocumentsByCrisisCode(String crisisCode, Integer count) throws Exception { try { + logger.info("Getting human labelled documents by crisisCode: "+crisisCode); TaggerResponseWrapper labeledDataList = taggerService.getHumanLabeledDocumentsByCrisisCode(crisisCode, count); return getUIWrapper(labeledDataList, true); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while getting human labelled documents by crisisCode: "+crisisCode +"/t"+e.getStackTrace()); return getUIWrapper(false, "Error in fetching human labeled documents"); } } @@ -915,10 +928,11 @@ public Map getHumanLabeledDocumentsByCrisisCode(String crisisCode @ResponseBody public Map getHumanLabeledDocumentsByCrisisIDUserID(Long crisisID, Long userID, Integer count) throws Exception { try { + logger.info("Getting human labelled documents by crisisID: "+crisisID +" & userID: "+userID); TaggerResponseWrapper labeledDataList = taggerService.getHumanLabeledDocumentsByCrisisIDUserID(crisisID, userID, count); return getUIWrapper(labeledDataList, true); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while getting human labelled documents by crisisID: "+crisisID +" & userID: "+userID+"/t"+e.getStackTrace()); return getUIWrapper(false, "Error in fetching human labeled documents"); } } @@ -927,10 +941,11 @@ public Map getHumanLabeledDocumentsByCrisisIDUserID(Long crisisID @ResponseBody public Map getHumanLabeledDocumentsByCrisisIDUserName(Long crisisID, String userName, Integer count) throws Exception { try { + logger.info("Getting human labelled documents by crisisID: "+crisisID +" & userName: "+userName); TaggerResponseWrapper labeledDataList = taggerService.getHumanLabeledDocumentsByCrisisIDUserName(crisisID, userName, count); return getUIWrapper(labeledDataList, true); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while getting human labelled documents by crisisID: "+crisisID +" & userName: "+userName+"/t"+e.getStackTrace()); return getUIWrapper(false, "Error in fetching human labeled documents"); } } @@ -960,7 +975,7 @@ public Map downloadHumanLabeledDocuments(String queryString, return getUIWrapper(downloadLink, false); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while downloading human labelled documents for crisis: "+crisisCode + " , fileType: "+fileType + " & queryString: "+queryString +"/t"+e.getStackTrace()); return getUIWrapper(false, "Error in getting download link for human labeled documents"); } } @@ -978,7 +993,7 @@ public Map updateMicromapperEnabled(@RequestParam Boolean isMicro return getUIWrapper(false, "Something wrong while updating isMicromapperEnabled"); } } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while updating isMicromapperEnabled flag for crisis: "+code ,e); return getUIWrapper(false, "Unable to update micromapperEnabled for collection, code = " + code); } } @@ -1006,7 +1021,7 @@ public Map sendEmail(@RequestParam String url, @RequestParam Str return getUIWrapper(false, "Sending Email Failed"); } }catch (Exception e) { - e.printStackTrace(); + logger.error("Error while sending emails to develoopers", e); return getUIWrapper(false, "Sending Email Failed"); } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/UITemplateController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/UITemplateController.java index 422084f5b..19aed787f 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/UITemplateController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/UITemplateController.java @@ -1,22 +1,25 @@ package qa.qcri.aidr.manager.controller; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; + import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.propertyeditors.CustomDateEditor; import org.springframework.stereotype.Controller; import org.springframework.web.bind.WebDataBinder; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.InitBinder; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; + import qa.qcri.aidr.manager.dto.UITemplateRequest; import qa.qcri.aidr.manager.exception.AidrException; import qa.qcri.aidr.manager.service.CollectionService; -import qa.qcri.aidr.manager.service.TaggerService; import qa.qcri.aidr.manager.service.UITemplateService; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.Map; - /** * Created with IntelliJ IDEA. * User: jlucas @@ -56,7 +59,7 @@ public Map updateTemplate(UITemplateRequest uiTemplateRequest) { return getUIWrapper(false, "Error while creating new ui template in uiTemplate"); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while updating UI Template", e); return getUIWrapper(false, e.getMessage()); } } @@ -77,7 +80,7 @@ public Map getTemplate(long crisisID) { } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.error("Error while fetching templates by crisisID: "+crisisID, e); return getUIWrapper(false, e.getMessage()); } } @@ -88,14 +91,13 @@ public Map getCrisisChildren(@RequestParam Integer id) throws Exc try { logger.info("Get Assignable Task is started with crisis id: " + id); - String userName = getAuthenticatedUserName(); + getAuthenticatedUserName(); String sVar = uiTemplateService.getCrisisChildrenElement(id); - // logger.info("sVar : " + sVar); return getUIWrapper(sVar, true); } catch (AidrException e) { - e.printStackTrace(); - return getUIWrapper(e.getMessage(), false); + logger.error("Error while getting crisis children elements for crisisID: "+id, e); + return getUIWrapper(e.getMessage(), false); } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/UserController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/UserController.java index 026389207..17c33ce40 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/UserController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/UserController.java @@ -36,7 +36,7 @@ public Map getCurrentUserRoles() throws Exception { UserEntity entity = getAuthenticatedUser(); return getUIWrapper(entity, true); }catch(Exception e){ - String msg = "Error while getting current user"; + String msg = "Error while getting current user "; logger.error(msg, e); return getUIWrapper(false, msg); } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaggerCrisisRequest.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaggerCrisisRequest.java index cb3d85d26..f86a13a01 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaggerCrisisRequest.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaggerCrisisRequest.java @@ -1,10 +1,6 @@ package qa.qcri.aidr.manager.dto; -import java.util.ArrayList; -import java.util.List; - import qa.qcri.aidr.dbmanager.dto.CrisisDTO; -import qa.qcri.aidr.dbmanager.dto.ModelFamilyDTO; public class TaggerCrisisRequest { diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaggerModel.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaggerModel.java index ca108d6e3..39b9d3626 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaggerModel.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaggerModel.java @@ -1,6 +1,5 @@ package qa.qcri.aidr.manager.dto; -import qa.qcri.aidr.dbmanager.dto.ModelDTO; public class TaggerModel { diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaskAnswer.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaskAnswer.java index b4806dcf0..db0ef4074 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaskAnswer.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/dto/TaskAnswer.java @@ -1,6 +1,5 @@ package qa.qcri.aidr.manager.dto; -import qa.qcri.aidr.dbmanager.dto.TaskAnswerDTO; public class TaskAnswer { diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/hibernateEntities/AidrAuthenticateToken.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/hibernateEntities/AidrAuthenticateToken.java index 53cd8b6d5..7391eedc5 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/hibernateEntities/AidrAuthenticateToken.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/hibernateEntities/AidrAuthenticateToken.java @@ -1,12 +1,16 @@ package qa.qcri.aidr.manager.hibernateEntities; -import javax.persistence.*; -import javax.validation.constraints.NotNull; +import java.io.Serializable; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; -import java.io.Serializable; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; @Entity diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/repository/impl/CollectionRepositoryImpl.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/repository/impl/CollectionRepositoryImpl.java index 192b714fa..f35cd99bf 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/repository/impl/CollectionRepositoryImpl.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/repository/impl/CollectionRepositoryImpl.java @@ -9,6 +9,8 @@ import java.util.Collections; import java.util.List; + +import org.apache.log4j.Logger; //import org.apache.log4j.Logger; import org.hibernate.Criteria; import org.hibernate.HibernateException; @@ -32,7 +34,7 @@ @Repository("collectionRepository") public class CollectionRepositoryImpl extends GenericRepositoryImpl implements CollectionRepository{ - //private Logger logger = Logger.getLogger(getClass()); + private Logger logger = Logger.getLogger(CollectionRepositoryImpl.class); @SuppressWarnings("unchecked") @Override @@ -450,7 +452,7 @@ public AidrCollection findByCode(String code) { try { return (AidrCollection) criteria.uniqueResult(); } catch (HibernateException e) { - e.printStackTrace(); + logger.error("Hibernate exception while finding a collection by code: "+code + "/t"+e.getStackTrace()); return null; } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/repository/impl/UserRepositoryImpl.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/repository/impl/UserRepositoryImpl.java index 655d28f6c..7ebdd9e9e 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/repository/impl/UserRepositoryImpl.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/repository/impl/UserRepositoryImpl.java @@ -6,15 +6,12 @@ import org.hibernate.Criteria; import org.hibernate.ScrollableResults; -import org.hibernate.criterion.LogicalExpression; import org.hibernate.criterion.Restrictions; import org.springframework.stereotype.Repository; - import org.springframework.util.StringUtils; -import qa.qcri.aidr.manager.hibernateEntities.AidrCollection; + import qa.qcri.aidr.manager.hibernateEntities.UserEntity; import qa.qcri.aidr.manager.repository.UserRepository; -import qa.qcri.aidr.manager.util.CollectionStatus; @Repository("userRepository") public class UserRepositoryImpl extends GenericRepositoryImpl implements UserRepository { diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/security/authentication/AjaxTimeoutRedirectFilter.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/security/authentication/AjaxTimeoutRedirectFilter.java index 0eb176fb1..19673f866 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/security/authentication/AjaxTimeoutRedirectFilter.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/security/authentication/AjaxTimeoutRedirectFilter.java @@ -61,6 +61,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha throw ase; } } else { + logger.error("AuthenticationException"); throw ase; } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/CollectionLogServiceImpl.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/CollectionLogServiceImpl.java index dc0778aa9..acf47c61f 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/CollectionLogServiceImpl.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/CollectionLogServiceImpl.java @@ -1,15 +1,7 @@ package qa.qcri.aidr.manager.service.impl; -import qa.qcri.aidr.manager.dto.CollectionLogDataResponse; -import qa.qcri.aidr.manager.exception.AidrException; -import qa.qcri.aidr.manager.hibernateEntities.AidrCollectionLog; -import qa.qcri.aidr.manager.repository.CollectionLogRepository; -import qa.qcri.aidr.manager.service.CollectionLogService; - - - - - +import java.util.List; +import java.util.Map; //import com.sun.jersey.api.client.Client; // gf 3 way //import com.sun.jersey.api.client.ClientResponse; @@ -17,21 +9,21 @@ import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.log4j.Logger; +import org.glassfish.jersey.jackson.JacksonFeature; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import javax.ws.rs.core.MediaType; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.glassfish.jersey.jackson.JacksonFeature; +import qa.qcri.aidr.manager.dto.CollectionLogDataResponse; +import qa.qcri.aidr.manager.exception.AidrException; +import qa.qcri.aidr.manager.hibernateEntities.AidrCollectionLog; +import qa.qcri.aidr.manager.repository.CollectionLogRepository; +import qa.qcri.aidr.manager.service.CollectionLogService; @Service("collectionLogService") public class CollectionLogServiceImpl implements CollectionLogService { @@ -113,6 +105,7 @@ public Map generateCSVLink(String code) throws AidrException { return ""; }*/ } catch (Exception e) { + logger.error("Error while generating CSV link", e); throw new AidrException("[generateCSVLink] Error while generating CSV link in Persister", e); } } @@ -134,6 +127,7 @@ public Map generateTweetIdsLink(String code) throws AidrExceptio return ""; }*/ } catch (Exception e) { + logger.error("Error while generating Tweet Ids link", e); throw new AidrException("[generateTweetIdsLink] Error while generating Tweet Ids link in Persister", e); } } @@ -155,6 +149,7 @@ public Map generateJSONLink(String code, String jsonType) throws return ""; }*/ } catch (Exception e) { + logger.error("Error while generating JSON download link",e); throw new AidrException("[generateJSONLink] Error while generating JSON download link in Persister", e); } } @@ -177,6 +172,7 @@ public Map generateJsonTweetIdsLink(String code, String jsonType return ""; }*/ } catch (Exception e) { + logger.error("Error while generating JSON Tweet Ids download link",e); throw new AidrException("[generateJsonTweetIdsLink] Error while generating JSON Tweet Ids download link in Persister", e); } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/CollectionServiceImpl.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/CollectionServiceImpl.java index b01932d79..2ad890506 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/CollectionServiceImpl.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/CollectionServiceImpl.java @@ -63,8 +63,6 @@ public class CollectionServiceImpl implements CollectionService { private AuthenticateTokenRepository authenticateTokenRepository; //@Autowired // gf 3 way - //private Client client; - //private Client client = ClientBuilder.newClient(); @Value("${fetchMainUrl}") private String fetchMainUrl; @Value("${twitter.consumerKey}") @@ -326,7 +324,7 @@ public boolean pingCollector() throws AidrException { return false; } } catch (Exception e) { - throw new AidrException("Error while Getting training data for Crisis and Model.", e); + throw new AidrException("Error while pinging the collector.", e); } } @@ -549,27 +547,31 @@ public String getFollowTwitterIDs(String followList, String userName) { userIdList = new long[userList.size()]; int i = 0; int j = 0; - System.out.println("Received string: " + followList + ", Split follow string: " + userList); + logger.info("Received string: " + followList + ", Split follow string: " + userList); + //System.out.println("Received string: " + followList + ", Split follow string: " + userList); for (String user: userList) { - System.out.println("Looking at follow data: " + user); + logger.info("Looking at follow data: " + user); + //System.out.println("Looking at follow data: " + user); if (StringUtils.isNumeric(user.trim())) { try { userIdList[j] = Long.parseLong(user.trim()); - System.out.println("Going to fetch twitter userData for the following twitterID: " + userIdList[j]); + logger.info("Going to fetch twitter userData for the following twitterID: " + userIdList[j]); + //System.out.println("Going to fetch twitter userData for the following twitterID: " + userIdList[j]); ++j; } catch (Exception ex) { logger.error("Exception in parsing string to number: ", ex); } } else { userNameList[i] = user.trim(); - System.out.println("Going to fetch twitter userData for the following screen name: " + userNameList[i]); + logger.info("Going to fetch twitter userData for the following screen name: " + userNameList[i]); + //System.out.println("Going to fetch twitter userData for the following screen name: " + userNameList[i]); ++i; } } userNameList = ArrayUtils.subarray(userNameList, 0, i); userIdList = ArrayUtils.subarray(userIdList, 0, j); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while getting follow twitter Ids",e); } } List dataList = new ArrayList(); @@ -586,7 +588,8 @@ public String getFollowTwitterIDs(String followList, String userName) { followIDs.append(u.getId()).append(","); } followIDs.deleteCharAt(followIDs.lastIndexOf(",")); - System.out.println("Created follow twitterID list: " + followIDs.toString()); + logger.info("Created follow twitterID list: " + followIDs.toString()); + //System.out.println("Created follow twitterID list: " + followIDs.toString()); return followIDs.toString(); } else { @@ -614,13 +617,16 @@ public String getFollowTwitterScreenNames(String followList, String userName) { try { userIdList = new long[userList.size()]; int j = 0; - System.out.println("Received string: " + followList + ", Split follow string: " + userList); + logger.info("Received string: " + followList + ", Split follow string: " + userList); + //System.out.println("Received string: " + followList + ", Split follow string: " + userList); for (String user: userList) { - System.out.println("Looking at follow data: " + user); + logger.info("Looking at follow data: " + user); + //System.out.println("Looking at follow data: " + user); if (StringUtils.isNumeric(user.trim())) { try { userIdList[j] = Long.parseLong(user.trim()); - System.out.println("Going to fetch twitter userData for the following twitterID: " + userIdList[j]); + logger.info("Going to fetch twitter userData for the following twitterID: " + userIdList[j]); + //System.out.println("Going to fetch twitter userData for the following twitterID: " + userIdList[j]); ++j; } catch (Exception ex) { logger.error("Exception in parsing string to number: ", ex); @@ -643,7 +649,8 @@ public String getFollowTwitterScreenNames(String followList, String userName) { followScreenNames.append(u.getScreenName()).append(","); } followScreenNames.deleteCharAt(followScreenNames.lastIndexOf(",")); - System.out.println("Created follow twitterID list: " + followScreenNames.toString()); + logger.info("Created follow twitterID list: " + followScreenNames.toString()); + //System.out.println("Created follow twitterID list: " + followScreenNames.toString()); return followScreenNames.toString(); } else { @@ -664,10 +671,11 @@ private List getUserDataFromScreenName(String[] userNameList, String userN twitter.setOAuthAccessToken(accessToken); ResponseList list = twitter.lookupUsers(userNameList); - System.out.println("Successfully looked up in Twitter by screen name, size of list: " + list.size()); + logger.info("Successfully looked up in Twitter by screen name, size of list: " + list.size()); + //System.out.println("Successfully looked up in Twitter by screen name, size of list: " + list.size()); return (list != null ? list : new ArrayList()); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while getting user Data from screen Name for user: "+userName,e); } } return new ArrayList(); @@ -682,10 +690,11 @@ private List getUserDataFromTwitterID(long[] userIdList, String userName) twitter.setOAuthAccessToken(accessToken); ResponseList list = twitter.lookupUsers(userIdList); - System.out.println("Successfully looked up in Twitter by ID, size of list: " + list.size()); + logger.info("Successfully looked up in Twitter by ID, size of list: " + list.size()); + //System.out.println("Successfully looked up in Twitter by ID, size of list: " + list.size()); return (list != null ? list : new ArrayList()); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while getting user Data from TwitterId for user: "+userName,e); } } return new ArrayList(); diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/TaggerServiceImpl.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/TaggerServiceImpl.java index 1c5e98794..e0d529953 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/TaggerServiceImpl.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/TaggerServiceImpl.java @@ -1,5 +1,20 @@ package qa.qcri.aidr.manager.service.impl; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Form; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + import org.apache.log4j.Logger; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonNode; @@ -16,23 +31,39 @@ import qa.qcri.aidr.dbmanager.dto.NominalLabelDTO; import qa.qcri.aidr.dbmanager.dto.UsersDTO; import qa.qcri.aidr.dbmanager.dto.taggerapi.TrainingDataDTO; -import qa.qcri.aidr.manager.dto.*; +import qa.qcri.aidr.manager.dto.ModelHistoryWrapper; +import qa.qcri.aidr.manager.dto.PingResponse; +import qa.qcri.aidr.manager.dto.TaggerAllCrisesResponse; +import qa.qcri.aidr.manager.dto.TaggerAllCrisesTypesResponse; +import qa.qcri.aidr.manager.dto.TaggerAttribute; +import qa.qcri.aidr.manager.dto.TaggerCrisesAttribute; +import qa.qcri.aidr.manager.dto.TaggerCrisis; +import qa.qcri.aidr.manager.dto.TaggerCrisisAttributesResponse; +import qa.qcri.aidr.manager.dto.TaggerCrisisExist; +import qa.qcri.aidr.manager.dto.TaggerCrisisModelsResponse; +import qa.qcri.aidr.manager.dto.TaggerCrisisRequest; +import qa.qcri.aidr.manager.dto.TaggerCrisisType; +import qa.qcri.aidr.manager.dto.TaggerLabel; +import qa.qcri.aidr.manager.dto.TaggerLabelRequest; +import qa.qcri.aidr.manager.dto.TaggerModel; +import qa.qcri.aidr.manager.dto.TaggerModelFamily; +import qa.qcri.aidr.manager.dto.TaggerModelLabelsResponse; +import qa.qcri.aidr.manager.dto.TaggerModelNominalLabel; +import qa.qcri.aidr.manager.dto.TaggerResponseWrapper; +import qa.qcri.aidr.manager.dto.TaggerStatusResponse; +import qa.qcri.aidr.manager.dto.TaggerUser; +import qa.qcri.aidr.manager.dto.TaggersForCodes; +import qa.qcri.aidr.manager.dto.TaggersForCollectionsRequest; +import qa.qcri.aidr.manager.dto.TaggersForCollectionsResponse; +import qa.qcri.aidr.manager.dto.TaskAnswer; +import qa.qcri.aidr.manager.dto.TrainingDataRequest; +import qa.qcri.aidr.manager.dto.ValueModel; import qa.qcri.aidr.manager.exception.AidrException; import qa.qcri.aidr.manager.hibernateEntities.AidrCollection; import qa.qcri.aidr.manager.service.TaggerService; import qa.qcri.aidr.manager.util.ManagerConfigurationProperty; import qa.qcri.aidr.manager.util.ManagerConfigurator; -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.Form; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import java.util.*; - //import com.sun.jersey.api.client.Client; //import com.sun.jersey.api.client.ClientResponse; //import com.sun.jersey.api.client.WebResource; @@ -101,7 +132,7 @@ public List getAllCrisisTypes() throws AidrException { return crisesTypesResponse.getCrisisTypes(); } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting all crisis from Tagger", e); throw new AidrException( "Error while getting all crisis from Tagger", e); } @@ -139,7 +170,7 @@ public List getCrisesByUserId(Integer userId) return taggerAllCrisesResponse.getCrisises(); } catch (Exception e) { - logger.info("exception", e); + logger.error("Exception while fetching crisis by userId: "+userId, e); throw new AidrException( "No collection is enabled for Tagger. Please enable tagger for one of your collections.", e); @@ -166,7 +197,7 @@ public String createNewCrises(TaggerCrisisRequest crisis) return clientResponse.readEntity(String.class); } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while creating new crisis: "+crisis.getName(), e); throw new AidrException( "Error while creating new crises in Tagger", e); } @@ -208,7 +239,7 @@ public Collection getAttributesForCrises(Integer crisisID, return convertTaggerCrisesAttributeToDTO( crisisAttributesResponse.getCrisisAttributes(), userId); } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting all attributes for crisis from Tagger for crisisID: "+crisisID, e); throw new AidrException( "Error while getting all attributes for crisis from Tagger", e); @@ -239,7 +270,7 @@ public Map countCollectionsClassifiers( return rv; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting amount of classifiers by collection codes in Tagger", e); throw new AidrException( "Error while getting amount of classifiers by collection codes in Tagger", e); @@ -269,15 +300,13 @@ public TaggerCrisisExist isCrisesExist(String code) throws AidrException { if (crisisExist.getCrisisId() != null) { logger.info("Response from Tagger-API for Crises with the code " - + code - + ", found crisisID = " - + crisisExist.getCrisisId()); + + code+ ", found crisisID = "+ crisisExist.getCrisisId()); return crisisExist; } else { return null; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while checking if crisis exist in Tagger for collection: "+code, e); throw new AidrException( "Error while checking if crisis exist in Tagger", e); } @@ -317,7 +346,7 @@ public Integer isUserExistsByUsername(String userName) throws AidrException { return null; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while checking if user: "+userName +"exist", e); throw new AidrException( "Error while checking if user exist in Tagger", e); } @@ -360,7 +389,7 @@ public Integer addNewUser(TaggerUser taggerUser) throws AidrException { return null; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while adding new user to Tagger", e); throw new AidrException("Error while adding new user to Tagger", e); } } @@ -414,7 +443,7 @@ public Integer addAttributeToCrisis(TaggerModelFamily modelFamily) return null; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while adding attribute to crises", e); throw new AidrException("Error while adding attribute to crises", e); } } @@ -454,7 +483,7 @@ public TaggerCrisis getCrisesByCode(String code) throws AidrException { } return null; } catch (Exception e) { - logger.info("exception: ", e); + logger.error("Error while getting crisis by code for collection: "+code, e); return null; // throw new // AidrException("Error while getting crisis by code from Tagger", @@ -493,9 +522,9 @@ public TaggerCrisis updateCode(TaggerCrisis crisis) throws AidrException { } return crisis; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while updating crisis: "+crisis.getCode(), e); throw new AidrException( - "Error while getting crisis by code from Tagger", e); + "Error while updating crisis", e); } } @@ -549,7 +578,7 @@ public List getModelsForCrisis(Integer crisisID) } return null; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting all models for crisisId:"+crisisID, e); throw new AidrException( "Error while getting all models for crisis from Tagger", e); } @@ -586,7 +615,7 @@ public TaggerAttribute createNewAttribute(TaggerAttribute attribute) } return newAttribute; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while creating new attribute", e); throw new AidrException( "Error while creating new attribute in Tagger", e); } @@ -620,7 +649,7 @@ public TaggerAttribute getAttributeInfo(Integer id) throws AidrException { } return response; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting attribute info for attribute: "+id, e); throw new AidrException( "Error while getting attribute from Tagger", e); } @@ -650,7 +679,7 @@ public TaggerLabel getLabelInfo(Integer id) throws AidrException { } return null; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting label info for labelId: "+id, e); throw new AidrException("Error while getting label from Tagger", e); } } @@ -685,7 +714,7 @@ public boolean deleteAttribute(Integer id) throws AidrException { } return false; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while deleting a attribute for attributeId: "+id, e); throw new AidrException("Error while deleting attribute in Tagger", e); } @@ -723,7 +752,7 @@ public boolean deleteTrainingExample(Integer id) throws AidrException { } return false; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while deleting document", e); throw new AidrException("Error while deleting document in Tagger", e); } @@ -763,7 +792,7 @@ public boolean removeAttributeFromCrises(Integer modelFamilyID) } return false; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while removing classifier from crisis ", e); throw new AidrException( "Error while removing classifier from crisis in Tagger", e); } @@ -801,7 +830,7 @@ public TaggerAttribute updateAttribute(TaggerAttribute attribute) return attribute; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while updating attribute: "+attribute.getCode(), e); throw new AidrException("Error while updating attribute in Tagger", e); } @@ -842,7 +871,7 @@ public TaggerLabel updateLabel(TaggerLabelRequest label) return updatedLabel; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while updating label: "+label.getName(), e); throw new AidrException("Error while updating label in Tagger", e); } } @@ -876,11 +905,12 @@ public TaggerLabel createNewLabel(TaggerLabelRequest label) + " was created in Tagger"); return response; } else { + logger.error("Error while creating new label: "+label.getName()); throw new AidrException( "Error while creating new label in Tagger"); } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while creating new label: "+label.getName(), e); throw new AidrException("Error while creating new label in Tagger", e); } @@ -911,7 +941,7 @@ public TaggerAttribute attributeExists(String code) throws AidrException { return null; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while checking if attribute: "+code+" exist", e); throw new AidrException( "Error while checking if attribute exist in Tagger", e); } @@ -959,7 +989,7 @@ public List getTrainingDataByModelIdAndCrisisId( return null; } } catch (Exception e) { - logger.info("exception", e); + logger.error(crisisId + " Error while Getting training data for Crisis and Model", e); throw new AidrException( "Error while Getting training data for Crisis and Model.", e); @@ -1002,7 +1032,7 @@ public String getAssignableTask(Integer id, String userName) return jsonResponse; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting Assignable Task in Tagger for user: "+userName, e); throw new AidrException( "Error while getting Assignable Task in Tagger", e); } @@ -1024,7 +1054,7 @@ public String getTemplateStatus(String code) throws AidrException { return jsonResponse; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting Template Status in Tagger for code:"+code, e); throw new AidrException( "Error while getting Template Status in Tagger", e); } @@ -1056,7 +1086,7 @@ public String skipTask(Integer id, String userName) throws AidrException { return jsonResponse; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while Skip Task operation for user: "+userName, e); throw new AidrException("Error while Skip Task operation", e); } } @@ -1083,7 +1113,7 @@ public boolean saveTaskAnswer(List taskAnswer) return clientResponse.getStatus() == 204; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while saving TaskAnswer in AIDRCrowdsourcing", e); return true; // throw new // AidrException("Error while saving TaskAnswer in AIDRCrowdsourcing", @@ -1117,7 +1147,7 @@ public String loadLatestTweets(String code, String constraints) return ""; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while generating loading latest tweets for the collection: "+code, e); throw new AidrException( "Error while generating Tweet Ids link in taggerPersister", e); @@ -1153,7 +1183,7 @@ public ModelHistoryWrapper getModelHistoryByModelFamilyID(Integer start, jsonResponse, ModelHistoryWrapper.class); return modelHistoryWrapper; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while Getting history records for Mode", e); throw new AidrException( "Error while Getting history records for Model.", e); } @@ -1194,7 +1224,7 @@ public List getAllLabelsForModel(Integer modelID, return modelLabelsResponse.getModelNominalLabelsDTO(); } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting all labels for model from Tagger for the crisis: "+crisisCode, e); throw new AidrException( "Error while getting all labels for model from Tagger", e); } @@ -1219,7 +1249,7 @@ public String getRetainingThreshold() throws AidrException { return jsonResponse; } catch (Exception e) { - logger.info("exception", e); + logger.error("Exception while getting training sampleCountThreshold", e); throw new AidrException("getRetainingThreshold : ", e); } @@ -1265,8 +1295,8 @@ public Map getTaggersForCollections( return Collections.emptyMap(); } } catch (Exception e) { - logger.info("exception", e); - throw new AidrException("Error while adding new user to Tagger", e); + logger.error("Error while getting taggers for collections", e); + throw new AidrException("Error while getting taggers for collections", e); } } @@ -1275,19 +1305,12 @@ public boolean pingTagger() throws AidrException { Client client = ClientBuilder.newBuilder() .register(JacksonFeature.class).build(); try { - // WebResource webResource = client.resource(taggerMainUrl + - // "/misc/ping"); WebTarget webResource = client.target(taggerMainUrl + "/misc/ping"); ObjectMapper objectMapper = JacksonWrapper.getObjectMapper(); - // ClientResponse clientResponse = - // webResource.type(MediaType.APPLICATION_JSON) - // .accept(MediaType.APPLICATION_JSON) - // .get(ClientResponse.class); Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).get(); - // String jsonResponse = clientResponse.getEntity(String.class); String jsonResponse = clientResponse.readEntity(String.class); PingResponse pingResponse = objectMapper.readValue(jsonResponse, @@ -1299,10 +1322,9 @@ public boolean pingTagger() throws AidrException { return false; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while pinging tagger", e); throw new AidrException( - "Error while Getting training data for Crisis and Model.", - e); + "Error while pinging tagger.",e); } } @@ -1311,20 +1333,13 @@ public boolean pingTrainer() throws AidrException { Client client = ClientBuilder.newBuilder() .register(JacksonFeature.class).build(); try { - // WebResource webResource = client.resource(crowdsourcingAPIMainUrl - // + "/util/ping/heartbeat"); WebTarget webResource = client.target(crowdsourcingAPIMainUrl + "/util/ping/heartbeat"); ObjectMapper objectMapper = JacksonWrapper.getObjectMapper(); - // ClientResponse clientResponse = - // webResource.type(MediaType.APPLICATION_JSON) - // .accept(MediaType.APPLICATION_JSON) - // .get(ClientResponse.class); Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).get(); - // String jsonResponse = clientResponse.getEntity(String.class); String jsonResponse = clientResponse.readEntity(String.class); PingResponse pingResponse = objectMapper.readValue(jsonResponse, @@ -1335,10 +1350,9 @@ public boolean pingTrainer() throws AidrException { return false; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while pinging trainer", e); throw new AidrException( - "Error while Getting training data for Crisis and Model.", - e); + "Error while pinging trainer.",e); } } @@ -1347,20 +1361,13 @@ public boolean pingAIDROutput() throws AidrException { Client client = ClientBuilder.newBuilder() .register(JacksonFeature.class).build(); try { - // WebResource webResource = client.resource(outputAPIMainUrl + - // "/manage/ping"); WebTarget webResource = client.target(outputAPIMainUrl + "/manage/ping"); ObjectMapper objectMapper = JacksonWrapper.getObjectMapper(); - // ClientResponse clientResponse = - // webResource.type(MediaType.APPLICATION_JSON) - // .accept(MediaType.APPLICATION_JSON) - // .get(ClientResponse.class); Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).get(); - // String jsonResponse = clientResponse.getEntity(String.class); String jsonResponse = clientResponse.readEntity(String.class); PingResponse pingResponse = objectMapper.readValue(jsonResponse, @@ -1372,9 +1379,9 @@ public boolean pingAIDROutput() throws AidrException { return false; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while pinging output", e); throw new AidrException( - "Error while Getting training data for Crisis and Model.", + "Error while pinging output", e); } } @@ -1402,10 +1409,9 @@ public boolean pingPersister() throws AidrException { return false; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while pinging persister", e); throw new AidrException( - "Error while Getting training data for Crisis and Model.", - e); + "Error while pinging persister",e); } } @@ -1421,9 +1427,7 @@ public Map generateCSVLink(String code) + "&exportLimit=100000"); Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).get(); - // String jsonResponse = clientResponse.readEntity(String.class); - Map jsonResponse = clientResponse - .readEntity(Map.class); + Map jsonResponse = clientResponse.readEntity(Map.class); return jsonResponse; /* * if (jsonResponse != null && @@ -1431,7 +1435,7 @@ public Map generateCSVLink(String code) * jsonResponse; } else { return ""; } */ } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while generating CSV link in Persister for collection: "+code, e); throw new AidrException( "[generateCSVLink] Error while generating CSV link in Persister", e); @@ -1447,18 +1451,16 @@ public Map generateTweetIdsLink(String code) System.out.println("[generateTweetIdsLink] Received request for code: " + code); try { - System.out.println("Invoked URL: " + persisterMainUrl + /*System.out.println("Invoked URL: " + persisterMainUrl + "/taggerPersister/genTweetIds?collectionCode=" + code - + "&downloadLimited=true"); + + "&downloadLimited=true");*/ WebTarget webResource = client.target(persisterMainUrl + "/taggerPersister/genTweetIds?collectionCode=" + code + "&downloadLimited=true"); Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).get(); - // String jsonResponse = clientResponse.readEntity(String.class); - Map jsonResponse = clientResponse - .readEntity(Map.class); + Map jsonResponse = clientResponse.readEntity(Map.class); logger.info("Returning from func: " + jsonResponse); return jsonResponse; /* @@ -1467,7 +1469,7 @@ public Map generateTweetIdsLink(String code) * jsonResponse; } else { return ""; } */ } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while generating Tweet Ids link in Persister for collection: "+code, e); throw new AidrException( "[generateTweetIdsLink] Error while generating Tweet Ids link in Persister", e); @@ -1486,8 +1488,7 @@ public Map generateJSONLink(String code, String jsonType) Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).get(); // String jsonResponse = clientResponse.readEntity(String.class); - Map jsonResponse = clientResponse - .readEntity(Map.class); + Map jsonResponse = clientResponse.readEntity(Map.class); return jsonResponse; /* * if (jsonResponse != null && @@ -1495,7 +1496,7 @@ public Map generateJSONLink(String code, String jsonType) * jsonResponse; } else { return ""; } */ } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while generating JSON download link in Persister for collection: "+code, e); throw new AidrException( "[generateJSONLink] Error while generating JSON download link in Persister", e); @@ -1508,32 +1509,19 @@ public Map generateJsonTweetIdsLink(String code, String jsonType) throws AidrException { Client client = ClientBuilder.newBuilder() .register(JacksonFeature.class).build(); - System.out - .println("[generateJsonTweetIdsLink] Received request for code: " - + code); try { - System.out.println("[generateJsonTweetIdsLink] Invoked URL: " - + persisterMainUrl - + "/taggerPersister/genJsonTweetIds?collectionCode=" + code - + "&downloadLimited=true&" + "&jsonType=" + jsonType); WebTarget webResource = client.target(persisterMainUrl + "/taggerPersister/genJsonTweetIds?collectionCode=" + code + "&downloadLimited=true&" + "&jsonType=" + jsonType); Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).get(); - // String jsonResponse = clientResponse.readEntity(String.class); Map jsonResponse = clientResponse .readEntity(Map.class); logger.info("Returning from func: " + jsonResponse); return jsonResponse; - /* - * if (jsonResponse != null && - * "http".equals(jsonResponse.substring(0, 4))) { return - * jsonResponse; } else { return ""; } - */ } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while generating JSON Tweet Ids download link in Persister for collection: "+code, e); throw new AidrException( "[generateJsonTweetIdsLink] Error while generating JSON Tweet Ids download link in Persister", e); @@ -1562,9 +1550,9 @@ public Map generateCSVFilteredLink(String code, * jsonResponse; } else { return ""; } */ } catch (Exception e) { - logger.info("exception", e); + logger.info("Error while generating csv filtered download link in Persister for collection: "+code, e); throw new AidrException( - "[generateCSVFilteredLink] Error while generating JSON download link in Persister", + "[generateCSVFilteredLink] Error while generating csv filtered download link in Persister for collection", e); } } @@ -1589,21 +1577,15 @@ public Map generateTweetIdsFilteredLink(String code, Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).post(Entity.json(queryString), Response.class); - // String jsonResponse = clientResponse.readEntity(String.class); Map jsonResponse = clientResponse .readEntity(Map.class); logger.info("Returning from func: " + jsonResponse); return jsonResponse; - /* - * if (jsonResponse != null && - * "http".equals(jsonResponse.substring(0, 4))) { return - * jsonResponse; } else { return ""; } - */ } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while generating filtererd Tweet Ids download link in Persister for collection: "+code, e); throw new AidrException( - "[generateTweetIdsFilteredLink] Error while generating JSON Tweet Ids download link in Persister", + "[generateTweetIdsFilteredLink] Error while generating filtererd Tweet Ids download link in Persister", e); } } @@ -1622,7 +1604,6 @@ public Map generateJSONFilteredLink(String code, Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).post(Entity.json(queryString), Response.class); - // String jsonResponse = clientResponse.readEntity(String.class); Map jsonResponse = clientResponse.readEntity(Map.class); return jsonResponse; /* @@ -1631,9 +1612,9 @@ public Map generateJSONFilteredLink(String code, * jsonResponse; } else { return ""; } */ } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while generating filtered JSON download link in Persister for collection: "+code, e); throw new AidrException( - "[generateJSONFilteredLink] Error while generating JSON download link in Persister", + "[generateJSONFilteredLink] Error while generating filtered JSON download link in Persister", e); } } @@ -1661,7 +1642,6 @@ public Map generateJsonTweetIdsFilteredLink(String code, Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).post(Entity.json(queryString), Response.class); - // String jsonResponse = clientResponse.readEntity(String.class); Map jsonResponse = clientResponse .readEntity(Map.class); @@ -1673,9 +1653,9 @@ public Map generateJsonTweetIdsFilteredLink(String code, * jsonResponse; } else { return ""; } */ } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while generating filtered JSON Tweet Ids download link in Persister for collection: "+code, e); throw new AidrException( - "[generateJsonTweetIdsFilteredLink] Error while generating JSON Tweet Ids download link in Persister", + "[generateJsonTweetIdsFilteredLink] Error while generating filtered JSON Tweet Ids download link in Persister", e); } } @@ -1730,7 +1710,7 @@ private int getCurrentRetrainingThreshold() throws Exception { return sampleCountThreshold; } catch (Exception e) { - logger.info("exception", e); + logger.error("Exception while getting CurrentRetrainingThreshold", e); return 50; } @@ -1743,8 +1723,6 @@ private void deletePybossaApp(Integer modelFamilyID) { System.out .print("removeAttributeFromCrises: starting ......................................"); - // WebResource webResource = client.resource(taggerMainUrl + - // "/modelfamily/" + modelFamilyID); WebTarget webResource = client.target(taggerMainUrl + "/modelfamily/" + modelFamilyID); @@ -1752,10 +1730,6 @@ private void deletePybossaApp(Integer modelFamilyID) { objectMapper .setSerializationInclusion(JsonSerialize.Inclusion.NON_NULL); - // ClientResponse clientResponse = - // webResource.type(MediaType.APPLICATION_JSON) - // .accept(MediaType.APPLICATION_JSON) - // .delete(ClientResponse.class); Response resp = webResource.request(MediaType.APPLICATION_JSON) .get(); String jsonResp = resp.readEntity(String.class); @@ -1773,10 +1747,6 @@ private void deletePybossaApp(Integer modelFamilyID) { + "/clientapp/delete/" + crisisCode + "/" + attributeCode); - // ClientResponse clientResponse = - // webResource.type(MediaType.APPLICATION_JSON) - // .accept(MediaType.APPLICATION_JSON) - // .get(ClientResponse.class); Response clientResp = webResource.request( MediaType.APPLICATION_JSON).get(); logger.info("deactivated - clientResponse : " + clientResp); @@ -1784,7 +1754,6 @@ private void deletePybossaApp(Integer modelFamilyID) { logger.info("No modelfamily found for id = " + modelFamilyID); } } catch (Exception e) { - logger.info("exception", e); logger.error("deactivated - deletePybossaApp : " + e); } } @@ -1795,7 +1764,7 @@ public String getAttributesAndLabelsByCrisisId(Integer id) Client client = ClientBuilder.newBuilder() .register(JacksonFeature.class).build(); try { - // http://scd1.qcri.org:8084/AIDRTrainerAPI/rest/crisis/id/117 + // http://example.com:8084/AIDRTrainerAPI/rest/crisis/id/1234 WebTarget webResource = client.target(crowdsourcingAPIMainUrl + "/crisis/id/" + id); @@ -1808,7 +1777,7 @@ public String getAttributesAndLabelsByCrisisId(Integer id) return jsonResponse; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting all nominal attributes and their labels for a given crisisID: "+id, e); throw new AidrException( "Error while getting all nominal attributes and their labels for a given crisisID", e); @@ -1843,9 +1812,9 @@ public int trashCollection(AidrCollection collection) throws Exception { retVal = 0; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while attempting trash a collection: "+collection.getCode(), e); throw new AidrException( - "Error while attempting /trash REST call for aidr_predict", + "Error while deleting a collection from aidr-predict database", e); } logger.info("[trashCollection] result of cleaning aidr-predict: " @@ -1877,7 +1846,7 @@ public int trashCollection(AidrCollection collection) throws Exception { return 0; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while attempting trash REST call for aidr_scheduler for collection: "+collection.getCode(), e); throw new AidrException( "Error while attempting trash REST call for aidr_scheduler", e); @@ -1912,7 +1881,7 @@ public int untrashCollection(String collectionCode) throws Exception { return 0; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while attempting /untrash REST call for collection: "+collectionCode, e); throw new AidrException( "Error while attempting /untrash REST call", e); } @@ -1928,11 +1897,11 @@ public String loadLatestTweetsWithCount(String code, int count) WebTarget webResource = client.target(outputAPIMainUrl + "/crisis/fetch/channel/filter/" + code + "?count=" + count); - System.out.println("[loadLatestTweetsWithCount] Invoking: " + /*System.out.println("[loadLatestTweetsWithCount] Invoking: " + outputAPIMainUrl + "/crisis/fetch/channel/filter/" + code + "?count=" + count); System.out.println("[loadLatestTweetsWithCount] constraints: " - + constraints); + + constraints);*/ Response clientResponse = webResource.request( MediaType.APPLICATION_JSON).post(Entity.json(constraints), Response.class); @@ -1942,18 +1911,18 @@ public String loadLatestTweetsWithCount(String code, int count) if (jsonResponse != null && (jsonResponse.startsWith("{") || jsonResponse .startsWith("["))) { - System.out + /*System.out .println("[loadLatestTweetsWithCount] jsonResponse for collection " - + code + ": " + jsonResponse); + + code + ": " + jsonResponse);*/ return jsonResponse; } else { - System.out + /*System.out .println("[loadLatestTweetsWithCount] jsonResponse for collection " - + code + ": \"\""); + + code + ": \"\"");*/ return ""; } } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while loadLatestTweetsWithCount for collection: "+code, e); throw new AidrException("Error while loadLatestTweetsWithCount", e); } } @@ -1986,7 +1955,8 @@ public TaggerResponseWrapper getHumanLabeledDocumentsByCrisisID( return dtoList; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting all human labeled documents for crisisID = " + + crisisID + " from Tagger", e); throw new AidrException( "Error while getting all human labeled documents for crisisID = " + crisisID + " from Tagger", e); @@ -2021,7 +1991,8 @@ public TaggerResponseWrapper getHumanLabeledDocumentsByCrisisCode( return dtoList; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting all human labeled documents for crisis code = " + + crisisCode + " from Tagger", e); throw new AidrException( "Error while getting all human labeled documents for crisis code = " + crisisCode + " from Tagger", e); @@ -2056,7 +2027,9 @@ public TaggerResponseWrapper getHumanLabeledDocumentsByCrisisIDUserID( return dtoList; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting all human labeled documents for crisisID = " + + crisisID + ", userId = " + userID + + " from Tagger", e); throw new AidrException( "Error while getting all human labeled documents for crisisID = " + crisisID + ", userId = " + userID @@ -2092,7 +2065,9 @@ public TaggerResponseWrapper getHumanLabeledDocumentsByCrisisIDUserName( return dtoList; } catch (Exception e) { - logger.info("exception", e); + logger.error("Error while getting all human labeled documents for crisisID = " + + crisisID + ", user name = " + userName + + " from Tagger", e); throw new AidrException( "Error while getting all human labeled documents for crisisID = " + crisisID + ", user name = " + userName @@ -2140,13 +2115,10 @@ public Map downloadHumanLabeledDocumentsByCrisisUserName( retVal.put("total", response.getTotal()); return retVal; } catch (Exception e) { - logger.info("exception", e); - throw new AidrException( - "Error while getting download link for human labeled documents for crisis code = " - + crisisCode - + ", user name = " - + userName - + " from Tagger", e); + logger.error("Error while getting download link for human labeled documents for crisis code = " + + crisisCode+ ", user name = "+ userName+ " from Tagger", e); + throw new AidrException("Error while getting download link for human labeled documents for crisis code = " + + crisisCode+ ", user name = "+ userName+ " from Tagger", e); } } @@ -2162,7 +2134,7 @@ public Map updateMicromapperEnabled(String code, Boolean isMicro Map jsonResponse = clientResponse.readEntity(Map.class); return jsonResponse; } catch (Exception e) { - logger.info("Exception while updating isMicromapperEnabled ", e); + logger.error("Exception while updating isMicromapperEnabled ", e); throw new AidrException("Exception while updating isMicromapperEnabled ", e); } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/UITemplateServiceImpl.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/UITemplateServiceImpl.java index 11ff23cc0..915a0819d 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/UITemplateServiceImpl.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/service/impl/UITemplateServiceImpl.java @@ -1,28 +1,24 @@ package qa.qcri.aidr.manager.service.impl; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + import org.apache.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.annotate.JsonSerialize; import org.glassfish.jersey.jackson.JacksonFeature; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import qa.qcri.aidr.common.code.JacksonWrapper; import qa.qcri.aidr.manager.dto.UITemplateRequest; -import qa.qcri.aidr.manager.dto.UITemplateResponse; import qa.qcri.aidr.manager.exception.AidrException; import qa.qcri.aidr.manager.service.UITemplateService; import qa.qcri.aidr.manager.util.CustomUITemplateLookup; -import javax.ws.rs.client.Client; -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import java.util.List; - /** * Created with IntelliJ IDEA. * User: jlucas @@ -93,6 +89,7 @@ public UITemplateRequest updateTemplate(UITemplateRequest uiTemplateRequest) thr } return null; } catch (Exception e) { + logger.error("Error while creating new template in Tagger", e); throw new AidrException("Error while creating new template in Tagger", e); } @@ -118,6 +115,7 @@ public String getTemplatesByCrisisID(long crisisID) throws AidrException { } return null; } catch (Exception e) { + logger.error("Error while creating new template in Tagger", e); throw new AidrException("Error while creating new template in Tagger", e); } } @@ -136,6 +134,7 @@ public String getCrisisChildrenElement(Integer id) throws AidrException { return jsonResponse; } catch (Exception e) { + logger.error("Error while getting CrisisChildrenElement in UITemplate", e); throw new AidrException("Error while getting CrisisChildrenElement in UITemplate", e); } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/social/security/SpringSocialAuthenticationFilter.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/social/security/SpringSocialAuthenticationFilter.java index 4939fd611..b64e824c1 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/social/security/SpringSocialAuthenticationFilter.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/social/security/SpringSocialAuthenticationFilter.java @@ -112,6 +112,7 @@ public Authentication attemptAuthentication(HttpServletRequest request, } else if (allowRepeatedAuthenticationAttempts && alreadyAuthenticatedUserId != null) { return SecurityContextHolder.getContext().getAuthentication(); } else { + logger.info("SpringSocialSecurity sign in details not found in session"); throw new InsufficientAuthenticationException( "SpringSocialSecurity sign in details not found in session"); } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/social/security/SpringSocialUserDetailService.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/social/security/SpringSocialUserDetailService.java index d66c4b996..65f2cc84d 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/social/security/SpringSocialUserDetailService.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/social/security/SpringSocialUserDetailService.java @@ -5,6 +5,7 @@ import javax.annotation.Resource; +import org.apache.log4j.Logger; import org.socialsignin.springsocial.security.api.SpringSocialProfile; import org.socialsignin.springsocial.security.signin.SpringSocialSecurityAuthenticationFactory; import org.socialsignin.springsocial.security.signup.SignUpService; @@ -33,6 +34,7 @@ @Qualifier("springSocialUserDetailsService") public class SpringSocialUserDetailService implements UserDetailsService { + private static Logger logger = Logger.getLogger(SpringSocialUserDetailService.class); @Autowired private UsersConnectionRepository usersConnectionRepository; @@ -65,6 +67,7 @@ public UserDetails loadUserByUsername(String userName)throws UsernameNotFoundExc return new User(userName, authentication.getCredentials().toString(), true, true, true, true,authorities); } else { + logger.info("UsernameNotFoundException for user: "+userName); throw new UsernameNotFoundException(userName); } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDataValidator.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDataValidator.java index 9f755f804..c4e4e0412 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDataValidator.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDataValidator.java @@ -1,15 +1,13 @@ package qa.qcri.aidr.manager.util; -import org.apache.log4j.Logger; +import java.io.IOException; + import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.map.ObjectMapper; -import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; -import java.io.IOException; - /** * Created with IntelliJ IDEA. * User: jlucas diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDateDeSerializer.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDateDeSerializer.java index 31a667089..37c423447 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDateDeSerializer.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDateDeSerializer.java @@ -5,6 +5,7 @@ import java.text.SimpleDateFormat; import java.util.Date; +import org.apache.log4j.Logger; import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.JsonProcessingException; import org.codehaus.jackson.map.DeserializationContext; @@ -12,20 +13,22 @@ public class JsonDateDeSerializer extends JsonDeserializer{ - private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - + private static Logger logger = Logger.getLogger(JsonDateDeSerializer.class); + private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + @Override public Date deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { - String date = jp.getText(); - if(date!= null && !date.equals("")){ - try { - return dateFormat.parse(date); - } catch (ParseException e) { - throw new RuntimeException(e); - } - } - return null; - + String date = jp.getText(); + if(date!= null && !date.equals("")){ + try { + return dateFormat.parse(date); + } catch (ParseException e) { + logger.error("ParfseException for the date:"+date,e); + throw new RuntimeException(e); + } + } + return null; + } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDateSerializer.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDateSerializer.java index d77c8d7f1..8990b58b6 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDateSerializer.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/JsonDateSerializer.java @@ -1,25 +1,30 @@ package qa.qcri.aidr.manager.util; -import org.codehaus.jackson.JsonGenerator; -import org.codehaus.jackson.JsonProcessingException; -import org.codehaus.jackson.map.JsonSerializer; -import org.codehaus.jackson.map.SerializerProvider; - import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; -public class JsonDateSerializer extends JsonSerializer { +import org.apache.log4j.Logger; +import org.codehaus.jackson.JsonGenerator; +import org.codehaus.jackson.map.JsonSerializer; +import org.codehaus.jackson.map.SerializerProvider; +public class JsonDateSerializer extends JsonSerializer { + private static Logger logger = Logger.getLogger(JsonDateSerializer.class); private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); @Override - public void serialize(Date date, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) - throws IOException, JsonProcessingException { + public void serialize(Date date, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException + { String formattedDate = null; if (date != null) { formattedDate = dateFormat.format(date); } - jsonGenerator.writeString(formattedDate); + try { + jsonGenerator.writeString(formattedDate); + } catch (IOException e) { + logger.error("Exception while generating the json for the string:"+formattedDate, e); + throw e; + } } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/ScheduledTask.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/ScheduledTask.java index bdf8c5cae..90475e9a8 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/ScheduledTask.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/util/ScheduledTask.java @@ -31,8 +31,7 @@ private void scheduledTaskUpdateCollections() { try { collections = collectionService.getRunningCollections(); } catch (Exception e) { - logger.error("Error while executing update collections scheduled task"); - logger.error("exception: ", e); + logger.error("Error while executing update collections scheduled task", e); taggerService.sendMailService("Error in ScheduledTask","Error while executing update collections scheduled task in ScheduledTask.scheduledTaskUpdateCollections"); return; } @@ -43,8 +42,7 @@ private void scheduledTaskUpdateCollections() { try { collectionService.statusByCollection(item); } catch (Exception e) { - logger.error("Error while updating collection with ID: " + item.getId()); - logger.error("exception: ", e); + logger.error("Error while updating collection with ID: " + item.getId(), e); taggerService.sendMailService("Error in ScheduledTask","Error while executing updating collection with ID: " + item.getId() +" in ScheduledTask.scheduledTaskUpdateCollections"); } } @@ -58,8 +56,7 @@ private void scheduledTaskStopCollections() { try { collections = collectionService.getRunningCollections(); } catch (Exception e) { - logger.error("Error while executing checking for collections running duration"); - logger.error("exception: ", e); + logger.error("Error while executing checking for collections running duration",e); taggerService.sendMailService("Error in ScheduledTask","Error while executing checking for collections running duration in ScheduledTask.scheduledTaskStopCollections"); return; } @@ -74,8 +71,7 @@ private void scheduledTaskStopCollections() { collectionService.stop(item.getId()); logger.info("Collection with ID: " + item.getId() + " was automatically stopped as it reached duration interval."); } catch (Exception e) { - logger.info("Error while stopping collection with ID: " + item.getId()); - logger.error("exception: ", e); + logger.info("Error while stopping collection with ID: " + item.getId(),e); taggerService.sendMailService("Error in ScheduledTask","Error while stopping collection with ID: " + item.getId() +" in ScheduledTask.scheduledTaskStopCollections"); } } From 0c2c31e71e6dac400d13b33e00e77d0eaa6e018f Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Sat, 1 Aug 2015 14:13:01 +0530 Subject: [PATCH 32/44] Commit for #97809356 : aidr-db-manager logging --- .../qa/qcri/aidr/dbmanager/dto/CrisisDTO.java | 13 +----- .../aidr/dbmanager/dto/CrisisTypeDTO.java | 5 ++- .../qcri/aidr/dbmanager/dto/DocumentDTO.java | 9 ++-- .../dto/HumanLabeledDocumentDTO.java | 5 ++- .../dto/HumanLabeledDocumentList.java | 7 ++- .../qa/qcri/aidr/dbmanager/dto/ModelDTO.java | 20 +++++++-- .../aidr/dbmanager/dto/ModelFamilyDTO.java | 1 - .../dbmanager/dto/ModelNominalLabelIdDTO.java | 4 ++ .../NominalAttributeDependentLabelIdDTO.java | 6 ++- .../dto/NominalLabelTrainingDataDTO.java | 4 ++ .../dto/NominalLabelTrainingDataIdDTO.java | 4 ++ .../aidr/dbmanager/dto/TaskAnswerDTO.java | 14 ++++-- .../aidr/dbmanager/dto/TaskAssignmentDTO.java | 9 +++- .../qa/qcri/aidr/dbmanager/dto/UsersDTO.java | 1 - .../HumanLabeledDocumentListWrapper.java | 11 ++--- .../dto/taggerapi/ModelDTOWrapper.java | 3 +- .../facade/impl/CoreDBServiceFacadeImp.java | 6 +-- .../facade/imp/CrisisResourceFacadeImp.java | 15 +++---- .../imp/CrisisTypeResourceFacadeImp.java | 8 ++-- .../CustomUiTemplateResourceFacadeImp.java | 11 ++++- ...DocumentNominalLabelResourceFacadeImp.java | 19 ++++---- .../facade/imp/DocumentResourceFacadeImp.java | 45 +++++++------------ .../imp/ModelFamilyResourceFacadeImp.java | 1 + .../ModelNominalLabelResourceFacadeImp.java | 24 +++++----- .../facade/imp/ModelResourceFacadeImp.java | 3 +- .../NominalAttributeResourceFacadeImp.java | 6 +-- .../imp/NominalLabelResourceFacadeImp.java | 6 +-- .../imp/SystemEventResourceFacadeImp.java | 17 ++++--- .../imp/TaskAnswerResourceFacadeImp.java | 3 +- .../imp/TaskAssignmentResourceFacadeImp.java | 5 +-- .../facade/imp/UsersResourceFacadeImp.java | 3 +- .../dbmanager/entities/misc/SystemEvent.java | 5 ++- 32 files changed, 156 insertions(+), 137 deletions(-) diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisDTO.java index 385ec50a7..f57f894f7 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisDTO.java @@ -109,7 +109,6 @@ public CrisisDTO(Long crisisID, String name, String code, boolean isTrashed, boo public CrisisDTO(Crisis crisis) throws PropertyNotSetException { if (crisis != null) { - //System.out.println("Crisis Hash code: " + crisis.hashCode()); this.setCrisisID(crisis.getCrisisId()); this.setName(crisis.getName()); @@ -126,22 +125,18 @@ public CrisisDTO(Crisis crisis) throws PropertyNotSetException { user.setUserId(crisis.getUsers().getUserId()); this.setUsersDTO(new UsersDTO(user)); } - //System.out.println("Done setting user DTO"); // Setting optional fields that were lazily initialized if (crisis.hasNominalAttributes()) { this.setNominalAttributesDTO(toNominalAttributeDTOList(crisis.getNominalAttributes())); } - //System.out.println("Done setting nominalAttributes DTO"); if (crisis.hasModelFamilies()) { this.setModelFamiliesDTO(toModelFamilyDTOList(crisis.getModelFamilies())); } - //System.out.println("Done setting modelfamily DTO"); if (crisis.hasDocuments()) { this.setDocumentsDTO(toDocumentDTOList(crisis.getDocuments())); } - //System.out.println("Done setting document DTO"); } else { - System.out.println("Entity = null in constructor"); + logger.error("Entity = null in constructor"); } } @@ -232,7 +227,6 @@ public void setModelFamiliesDTO(List modelFamiliesDTO) { } private List toDocumentDTOList(List list) { - try { if (list != null) { try { List dtoList = new ArrayList(); @@ -247,12 +241,9 @@ private List toDocumentDTOList(List list) { } return dtoList; } catch (Exception e) { - e.printStackTrace(); + logger.warn("Unable to wrap Document to DocumentDTO."); } } - } catch (Exception e) { - e.printStackTrace(); - } return null; } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisTypeDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisTypeDTO.java index ee0594401..ff2ef7541 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisTypeDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisTypeDTO.java @@ -6,6 +6,8 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + import qa.qcri.aidr.common.exception.PropertyNotSetException; import qa.qcri.aidr.dbmanager.entities.misc.Crisis; import qa.qcri.aidr.dbmanager.entities.misc.CrisisType; @@ -17,6 +19,7 @@ public class CrisisTypeDTO implements java.io.Serializable { private static final long serialVersionUID = 8074463052776843105L; + private Logger logger = Logger.getLogger(CrisisTypeDTO.class); @XmlElement private Long crisisTypeId; @@ -57,7 +60,7 @@ public CrisisTypeDTO(CrisisType crisisType) throws PropertyNotSetException { this.numberOfCrisisAssociated = crisisType.getCrisises() != null ? crisisType.getCrisises().size() : 0; } } else { - System.out.println("Entity = null in constructor"); + logger.error("Entity = null in constructor"); } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/DocumentDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/DocumentDTO.java index 86156bf12..895d7f509 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/DocumentDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/DocumentDTO.java @@ -9,6 +9,8 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import org.apache.log4j.Logger; + import qa.qcri.aidr.common.exception.PropertyNotSetException; import qa.qcri.aidr.dbmanager.entities.misc.Crisis; import qa.qcri.aidr.dbmanager.entities.misc.CrisisType; @@ -30,6 +32,7 @@ public class DocumentDTO implements Serializable { * */ private static final long serialVersionUID = 6833511192868020105L; + private Logger logger = Logger.getLogger(DocumentDTO.class); @XmlElement private Long documentID; @@ -117,7 +120,6 @@ public DocumentDTO(CrisisDTO crisis, boolean isEvaluationSet, public DocumentDTO(Document doc) throws PropertyNotSetException { if (doc != null) { - //System.out.println("Document Hash code: " + doc.hashCode()); if (doc.getDocumentId() != null) { this.setDocumentID(doc.getDocumentId()); } @@ -128,7 +130,6 @@ public DocumentDTO(Document doc) throws PropertyNotSetException { c.setCrisisId(doc.getCrisis().getCrisisId()); this.setCrisisDTO(new CrisisDTO(c)); } - //System.out.println("Done setting crisis DTO"); this.setDoctype(doc.getDoctype()); this.setGeoFeatures(doc.getGeoFeatures()); this.setIsEvaluationSet(doc.isIsEvaluationSet()); @@ -142,13 +143,11 @@ public DocumentDTO(Document doc) throws PropertyNotSetException { if (doc.hasDocumentNominalLabels()) { this.setDocumentNominalLabelsDTO(this.toDocumentNominalLabelDTOList(doc.getDocumentNominalLabels())); } - //System.out.println("Done setting DocumentNominalLabels DTO"); if (doc.hasTaskAssignments()) { this.setTaskAssignmentDTO(this.toTaskAssignmentDTOList(doc.getTaskAssignments())); } - //System.out.println("Done setting TaskAssignments DTO"); } else { - System.out.println("Entity = null in constructor"); + logger.error("Entity = null in constructor"); } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java index b6fbd56a6..bc86310a0 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java @@ -10,6 +10,8 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -26,6 +28,7 @@ public class HumanLabeledDocumentDTO implements Serializable { * */ private static final long serialVersionUID = -8284975892012345464L; + private Logger logger = Logger.getLogger(HumanLabeledDocumentDTO.class); @XmlElement private DocumentDTO doc; @@ -92,7 +95,7 @@ public String toJsonString() { String jsonString = jsonObject.toJson(this, HumanLabeledDocumentDTO.class); return jsonString; } catch (Exception e) { - e.printStackTrace(); + logger.warn("Error in parsing json to human labeled doc."); return null; } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java index 219c82d1e..d25105ab7 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java @@ -7,6 +7,8 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -19,7 +21,8 @@ public class HumanLabeledDocumentList implements Serializable { * */ private static final long serialVersionUID = 4864042221879747333L; - + private Logger logger = Logger.getLogger(HumanLabeledDocumentList.class); + @XmlElement private List items; @XmlElement private Integer total; @@ -66,7 +69,7 @@ public String toJsonString() { String jsonString = jsonObject.toJson(this, HumanLabeledDocumentList.class); return jsonString; } catch (Exception e) { - e.printStackTrace(); + logger.warn("Error in parsing json to human labeled doc list."); return null; } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelDTO.java index a09210465..a2622b62c 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelDTO.java @@ -2,8 +2,6 @@ // Generated Nov 24, 2014 4:55:08 PM by Hibernate Tools 4.0.0 package qa.qcri.aidr.dbmanager.dto; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - import java.io.Serializable; import java.util.ArrayList; import java.util.Date; @@ -12,12 +10,14 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + import qa.qcri.aidr.common.exception.PropertyNotSetException; -import qa.qcri.aidr.dbmanager.entities.misc.Crisis; import qa.qcri.aidr.dbmanager.entities.model.Model; import qa.qcri.aidr.dbmanager.entities.model.ModelFamily; import qa.qcri.aidr.dbmanager.entities.model.ModelNominalLabel; -import qa.qcri.aidr.dbmanager.entities.model.NominalAttribute; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @XmlRootElement @JsonIgnoreProperties(ignoreUnknown = true) @@ -27,6 +27,7 @@ public class ModelDTO implements Serializable { * */ private static final long serialVersionUID = 436159208494993271L; + private Logger logger = Logger.getLogger(ModelDTO.class); @XmlElement private Long modelId; @@ -122,8 +123,10 @@ public Long getModelId() { public void setModelId(Long modelId) { if (modelId == null) { + logger.error("modelID cannot be null"); throw new IllegalArgumentException("modelID cannot be null"); } else if (modelId.longValue() <= 0) { + logger.error("modelID cannot be zero or a negative number"); throw new IllegalArgumentException("modelID cannot be zero or a negative number"); } else { this.modelId = modelId; @@ -144,8 +147,10 @@ public double getAvgPrecision() { public void setAvgPrecision(Double avgPrecision) { if (avgPrecision == null) { + logger.error("Average percision cannot be null"); throw new IllegalArgumentException("Average percision cannot be null"); } else if (avgPrecision.doubleValue() < 0) { + logger.error("Average percision cannot be a negative number"); throw new IllegalArgumentException("Average percision cannot be a negative number"); } else { this.avgPrecision = avgPrecision; @@ -159,8 +164,10 @@ public double getAvgRecall() { public void setAvgRecall(Double avgRecall) { if (avgRecall == null) { + logger.error("Average recall cannot be null"); throw new IllegalArgumentException("Average recall cannot be null"); } else if (avgRecall.doubleValue() < 0) { + logger.error("Average recall cannot be a negative number"); throw new IllegalArgumentException("Average recall cannot be a negative number"); } else { this.avgRecall = avgRecall; @@ -173,8 +180,10 @@ public double getAvgAuc() { public void setAvgAuc(Double avgAuc) { if (avgAuc == null) { + logger.error("Average AUC cannot be null"); throw new IllegalArgumentException("Average AUC cannot be null"); } else if (avgAuc.doubleValue() < 0) { + logger.error("Average AUC cannot be a negative number"); throw new IllegalArgumentException("Average AUC cannot be a negative number"); } else { this.avgAuc = avgAuc; @@ -188,8 +197,10 @@ public int getTrainingCount() { public void setTrainingCount(Integer trainingCount) { if (trainingCount == null) { + logger.error("Average training cannot be null"); throw new IllegalArgumentException("Average training cannot be null"); } else if (trainingCount < 0) { + logger.error("Average training cannot be a negative number"); throw new IllegalArgumentException("Average training cannot be a negative number"); } else { this.trainingCount = trainingCount; @@ -202,6 +213,7 @@ public Date getTrainingTime() { public void setTrainingTime(Date trainingTime) { if (trainingTime == null) { + logger.error("Training time cannot be null"); throw new IllegalArgumentException("Training time cannot be null"); } else { this.trainingTime = trainingTime; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelFamilyDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelFamilyDTO.java index 0b69d7b6a..c9775669d 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelFamilyDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelFamilyDTO.java @@ -49,7 +49,6 @@ public ModelFamilyDTO() { public ModelFamilyDTO(ModelFamily model) throws PropertyNotSetException { if (model != null) { - //System.out.println("ModelFamily Hash code: " + model.hashCode()); this.setModelFamilyId(model.getModelFamilyId()); if (model.hasNominalAttribute()) { NominalAttribute na = new NominalAttribute(model.getNominalAttribute().getUsers(), diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelNominalLabelIdDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelNominalLabelIdDTO.java index c97011f40..a510fb107 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelNominalLabelIdDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelNominalLabelIdDTO.java @@ -5,6 +5,8 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + import qa.qcri.aidr.common.exception.PropertyNotSetException; import qa.qcri.aidr.dbmanager.entities.model.ModelNominalLabelId; @@ -18,6 +20,7 @@ public class ModelNominalLabelIdDTO implements Serializable { * */ private static final long serialVersionUID = -7966215874963999322L; + private Logger logger = Logger.getLogger(ModelNominalLabelIdDTO.class); @XmlElement private Long modelId; @@ -56,6 +59,7 @@ public void setNominalLabelId(Long nominalLabelId) { public ModelNominalLabelId toEntity() throws PropertyNotSetException { if (this.modelId == null || this.nominalLabelId == null) { + logger.error("Primary key not set!"); throw new PropertyNotSetException("Primary key not set!"); } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalAttributeDependentLabelIdDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalAttributeDependentLabelIdDTO.java index 8c0fab643..14e1f3941 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalAttributeDependentLabelIdDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalAttributeDependentLabelIdDTO.java @@ -5,6 +5,8 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + import qa.qcri.aidr.common.exception.PropertyNotSetException; import qa.qcri.aidr.dbmanager.entities.model.NominalAttributeDependentLabelId; @@ -17,7 +19,7 @@ public class NominalAttributeDependentLabelIdDTO implements Serializable { * */ private static final long serialVersionUID = 5910754106878087122L; - + private Logger logger = Logger.getLogger(NominalAttributeDependentLabelIdDTO.class); @XmlElement private Long nominalAttributeId; @@ -39,6 +41,7 @@ public NominalAttributeDependentLabelIdDTO(NominalAttributeDependentLabelId id) this.nominalAttributeId = id.getNominalAttributeId(); this.nominalLabelId = id.getNominalLabelId(); } else { + logger.error("Primary key not set!"); throw new PropertyNotSetException("Primary key not set!"); } } @@ -61,6 +64,7 @@ public void setNominalLabelId(Long nominalLabelId) { public NominalAttributeDependentLabelId toEntity() throws PropertyNotSetException { if (this.nominalAttributeId == null || this.nominalLabelId == null) { + logger.error("Primary key not set!"); throw new PropertyNotSetException("Primary key not set!"); } NominalAttributeDependentLabelId id = new NominalAttributeDependentLabelId(this.nominalAttributeId, this.nominalLabelId); diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataDTO.java index feed1e3be..4a63d8274 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataDTO.java @@ -5,6 +5,8 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import qa.qcri.aidr.common.exception.PropertyNotSetException; @@ -18,6 +20,7 @@ public class NominalLabelTrainingDataDTO implements Serializable { * */ private static final long serialVersionUID = -6001563823600724945L; + private Logger logger = Logger.getLogger(NominalLabelTrainingDataDTO.class); @XmlElement private NominalLabelTrainingDataIdDTO idDTO; @@ -46,6 +49,7 @@ public void setIdDTO(NominalLabelTrainingDataIdDTO idDTO) throws PropertyNotSetE if (idDTO != null) { this.idDTO = idDTO; } else { + logger.error("Primary key not set!"); throw new PropertyNotSetException("Primary key not set!"); } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataIdDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataIdDTO.java index e47c41dd3..654d6f6ab 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataIdDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataIdDTO.java @@ -5,6 +5,8 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import qa.qcri.aidr.common.exception.PropertyNotSetException; @@ -18,6 +20,7 @@ public class NominalLabelTrainingDataIdDTO implements Serializable { * */ private static final long serialVersionUID = 4397588498587349888L; + private Logger logger = Logger.getLogger(NominalLabelTrainingDataIdDTO.class); @XmlElement private Long documentId; @@ -45,6 +48,7 @@ public NominalLabelTrainingDataIdDTO(NominalLabelTrainingDataId id) throws Prope this.setNominalLabelId(id.getNominalLabelId()); this.setWordFeatures(id.getWordFeatures()); } else { + logger.error("Primary key not set!"); throw new PropertyNotSetException("Primary key not set!"); } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAnswerDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAnswerDTO.java index a66934f01..f9e65dc6d 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAnswerDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAnswerDTO.java @@ -8,7 +8,10 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + import qa.qcri.aidr.dbmanager.entities.task.TaskAnswer; @XmlRootElement @@ -20,6 +23,7 @@ public class TaskAnswerDTO implements Serializable { * */ private static final long serialVersionUID = 5525860415454423643L; + private Logger logger = Logger.getLogger(TaskAnswerDTO.class); @XmlElement private Long taskID; @@ -99,9 +103,10 @@ public Long getDocumentID() { } public void setDocumentID(Long documentID) { - if(documentID == null) + if(documentID == null) { + logger.error("documentID cannot be null"); throw new IllegalArgumentException("documentID cannot be null"); - + } this.documentID = documentID; } @@ -110,9 +115,10 @@ public Long getUserID() { } public void setUserID(Long userID) { - if(userID == null) + if(userID == null) { + logger.error("userID cannot be null"); throw new IllegalArgumentException("userID cannot be null"); - + } this.userID = userID; } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAssignmentDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAssignmentDTO.java index 0f7abce43..5dcc84d21 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAssignmentDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAssignmentDTO.java @@ -6,13 +6,14 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; -import com.fasterxml.jackson.annotation.JsonBackReference; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.log4j.Logger; import qa.qcri.aidr.common.exception.PropertyNotSetException; import qa.qcri.aidr.dbmanager.entities.task.Document; import qa.qcri.aidr.dbmanager.entities.task.TaskAssignment; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + @XmlRootElement @JsonIgnoreProperties(ignoreUnknown=true) public class TaskAssignmentDTO implements Serializable { @@ -20,6 +21,7 @@ public class TaskAssignmentDTO implements Serializable { * */ private static final long serialVersionUID = 6766433678441426060L; + private Logger logger = Logger.getLogger(TaskAssignmentDTO.class); @XmlElement private DocumentDTO document; @@ -92,6 +94,7 @@ public void setIdDTO(TaskAssignmentIdDTO idDTO) { if (idDTO != null) { this.idDTO = idDTO; } else { + logger.error("Primary key cannot be null"); throw new IllegalArgumentException("Primary key cannot be null"); } } @@ -102,6 +105,7 @@ public Long getDocumentID() { public void setDocumentID(Long documentID) throws PropertyNotSetException { if (documentID == null) { + logger.error("documentID cannot be null"); throw new IllegalArgumentException("documentID cannot be null"); } else{ @@ -116,6 +120,7 @@ public Long getUserID() { public void setUserID(Long userID) { if (userID == null) { + logger.error("userID cannot be null"); throw new IllegalArgumentException("userID cannot be null"); } else{ diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/UsersDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/UsersDTO.java index 3b4cadd16..e11485b2f 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/UsersDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/UsersDTO.java @@ -144,7 +144,6 @@ public UsersDTO(String name, String role) { public UsersDTO(Users user) throws PropertyNotSetException { if (user != null) { - //System.out.println("Users Hash code: " + user.hashCode()); this.name = user.getName(); this.role = user.getRole(); diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/HumanLabeledDocumentListWrapper.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/HumanLabeledDocumentListWrapper.java index 0b050ad99..ccdf00d38 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/HumanLabeledDocumentListWrapper.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/HumanLabeledDocumentListWrapper.java @@ -1,18 +1,18 @@ package qa.qcri.aidr.dbmanager.dto.taggerapi; import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; + +import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentList; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentList; - @JsonIgnoreProperties(ignoreUnknown=true) @XmlRootElement public class HumanLabeledDocumentListWrapper implements Serializable { @@ -21,6 +21,7 @@ public class HumanLabeledDocumentListWrapper implements Serializable { * */ private static final long serialVersionUID = -7413555749237029785L; + private Logger logger = Logger.getLogger(HumanLabeledDocumentListWrapper.class); @XmlElement HumanLabeledDocumentList dtoList; @@ -59,7 +60,7 @@ public String toJsonString() { String jsonString = jsonObject.toJson(this, HumanLabeledDocumentListWrapper.class); return jsonString; } catch (Exception e) { - e.printStackTrace(); + logger.warn("Error while parsing document list."); return null; } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/ModelDTOWrapper.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/ModelDTOWrapper.java index 10841a1d7..2ed21bdde 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/ModelDTOWrapper.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/ModelDTOWrapper.java @@ -4,12 +4,13 @@ */ package qa.qcri.aidr.dbmanager.dto.taggerapi; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.io.Serializable; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * * @author Imran diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java index 97c208e53..3ad371790 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java @@ -51,6 +51,7 @@ public EntityManager getEntityManager() { try { return em; } catch (Exception e) { + logger.error("getEntityManager failed", e); throw new HibernateException("getEntityManager failed"); } } @@ -66,7 +67,7 @@ public int setEntityManager(EntityManager em) { logger.info("Skipping setter, since EntityManager already initialized to :" + this.em); return 0; } catch (Exception e) { - logger.error("EntityManager setting exception : " + em); + logger.error("EntityManager setting exception : " + em, e); throw new HibernateException("setEntityManager failed"); } } @@ -126,7 +127,6 @@ public List getAll() { List fetchedList = new ArrayList(); try { fetchedList = criteria.list(); - //System.out.println("CoreDBServiceFacade: getAll fetched list size: " + fetchedList.size()); return fetchedList; } catch (Exception e) { logger.error("getAll failed", e); @@ -217,7 +217,6 @@ public List getByCriteriaWithAliasByOrder(Criterion criterion, String order, if(count != null && count > 0){ criteria.setMaxResults(count); } - //System.out.println("fetched List count = " + (fetchedList != null ? fetchedList.size() : null)); try { fetchedList = criteria.list(); return fetchedList; @@ -248,7 +247,6 @@ public List getByCriteriaWithInnerJoinByOrder(Criterion criterion, String ord if(count != null && count > 0){ criteria.setMaxResults(count); } - //System.out.println("fetched List count = " + (fetchedList != null ? fetchedList.size() : null)); try { fetchedList = criteria.list(); return fetchedList; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java index 98155781b..4bc589e55 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java @@ -26,7 +26,6 @@ import qa.qcri.aidr.dbmanager.ejb.remote.facade.CrisisResourceFacade; import qa.qcri.aidr.dbmanager.ejb.remote.facade.UsersResourceFacade; import qa.qcri.aidr.dbmanager.entities.misc.Crisis; -import qa.qcri.aidr.dbmanager.entities.misc.Users; @Stateless(name="CrisisResourceFacadeImp") public class CrisisResourceFacadeImp extends CoreDBServiceFacadeImp implements CrisisResourceFacade { @@ -72,7 +71,7 @@ public CrisisDTO addCrisis(CrisisDTO crisis) { em.refresh(c); return new CrisisDTO(c); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error in addCrisis for crisis code : " + crisis.getCode(), e); return null; } } @@ -112,7 +111,6 @@ public CrisisDTO getCrisisByCode(String code) throws PropertyNotSetException { @Override public CrisisDTO editCrisis(CrisisDTO crisis) throws PropertyNotSetException { - System.out.println("Received edit request for: " + crisis.getCrisisID() + ", " + crisis.getCode() + ", " + crisis.getName()); try { Crisis c = crisis.toEntity(); Crisis cr = getById(c.getCrisisId()); @@ -120,14 +118,13 @@ public CrisisDTO editCrisis(CrisisDTO crisis) throws PropertyNotSetException { cr = em.merge(c); em.flush(); em.refresh(cr); - System.out.println("Updated crisis: " + cr.getCode() + ", " + cr.getName() + ", " + cr.getCrisisType().getName()); return cr != null ? new CrisisDTO(cr) : null; } else { + logger.error("Not found"); throw new RuntimeException("Not found"); } } catch (Exception e) { - System.out.println("Exception in merging/updating crisis: " + crisis.getCrisisID()); - e.printStackTrace(); + logger.error("Exception in merging/updating crisis: " + crisis.getCrisisID(), e); } return null; @@ -135,19 +132,19 @@ public CrisisDTO editCrisis(CrisisDTO crisis) throws PropertyNotSetException { @Override public List getAllCrisis() throws PropertyNotSetException { - System.out.println("Received request for fetching all crisis!!!"); + logger.info("Received request for fetching all crisis!!!"); List dtoList = new ArrayList(); List crisisList = getAll(); if (crisisList != null && !crisisList.isEmpty()) { for (Crisis crisis : crisisList) { - System.out.println("Converting to DTO crisis: " + crisis.getCode() + ", " + crisis.getName() + ", " + crisis.getCrisisId() + logger.info("Converting to DTO crisis: " + crisis.getCode() + ", " + crisis.getName() + ", " + crisis.getCrisisId() + ", " + crisis.getUsers().getUserId() + ":" + crisis.getUsers().getName()); CrisisDTO dto = new CrisisDTO(crisis); dtoList.add(dto); } } - System.out.println("Done creating DTO list, size = " + dtoList.size()); + logger.info("Done creating DTO list, size = " + dtoList.size()); return dtoList; } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisTypeResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisTypeResourceFacadeImp.java index aac657130..3d69c81d3 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisTypeResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisTypeResourceFacadeImp.java @@ -61,21 +61,20 @@ public CrisisTypeDTO addCrisisType(CrisisTypeDTO crisisType) { @Override public CrisisTypeDTO editCrisisType(CrisisTypeDTO crisisType) throws PropertyNotSetException { - System.out.println("Received edit request for: " + crisisType.getName() + ", " + crisisType.getCrisisTypeId()); try { CrisisType cType = getById(crisisType.getCrisisTypeId()); if (cType != null) { cType = em.merge(crisisType.toEntity()); em.flush(); em.refresh(cType); - System.out.println("Updated crisisType: " + cType.getName() + ", " + cType.getCrisisTypeId()); + logger.info("Updated crisisType: " + cType.getName() + ", " + cType.getCrisisTypeId()); return cType != null ? new CrisisTypeDTO(cType) : null; } else { + logger.error("Not found"); throw new RuntimeException("Not found"); } } catch (Exception e) { - System.out.println("Exception in merging/updating crisisType: " + crisisType.getCrisisTypeId()); - e.printStackTrace(); + logger.error("Exception in merging/updating crisisType: " + crisisType.getCrisisTypeId(), e); } return null; } @@ -87,6 +86,7 @@ public Integer deleteCrisisType(Long id) { this.delete(crisisType); return 1; } else { + logger.error("CrisisType requested to be deleted does not exist! id = " + id); throw new RuntimeException("CrisisType requested to be deleted does not exist! id = " + id); } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CustomUiTemplateResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CustomUiTemplateResourceFacadeImp.java index 8ad9dcc77..57199eca2 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CustomUiTemplateResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CustomUiTemplateResourceFacadeImp.java @@ -43,6 +43,7 @@ public List getAllCustomUITemplateByCrisisID(long crisisID) } return dtoList; } catch (Exception e) { + logger.error("Error in getAllCustomUITemplateByCrisisID for crisisID : " + crisisID, e); return null; } } @@ -67,6 +68,7 @@ public List getCustomUITemplateBasedOnTypeByCrisisID(long c } return dtoList; } catch (Exception e) { + logger.error("Error in getCustomUITemplateBasedOnTypeByCrisisID for crisisID : " + crisisID); return null; } } @@ -90,6 +92,8 @@ public List getCustomUITemplateBasedOnTypeByCrisisIDAndAttr } return dtoList; } catch (Exception e) { + logger.error("Error in getCustomUITemplateBasedOnTypeByCrisisIDAndAttributeID for crisisID : " + crisisID + + " and attributeID : " + attributeID); return null; } } @@ -112,6 +116,8 @@ public List getCustomUITemplateByCrisisIDAndAttributeID(lon } return dtoList; } catch (Exception e) { + logger.error("Error in getCustomUITemplateByCrisisIDAndAttributeID " + + "crisisID : " + crisisID + " and attributeID : " + attributeID); return null; } } @@ -157,8 +163,9 @@ public int deleteCustomUiTemplateById(long customUiTemplateId){ Object managed = em.merge(customUiTemplateDTO.toEntity()); em.remove(managed); return 1; - } catch (Exception e) { - return 0; + } catch (Exception e) { + logger.warn("Warning! couldn't delete template by ID : " + customUiTemplateId); + return 0; } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentNominalLabelResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentNominalLabelResourceFacadeImp.java index b28d2ad37..80c33973f 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentNominalLabelResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentNominalLabelResourceFacadeImp.java @@ -69,10 +69,9 @@ public DocumentNominalLabelDTO addDocument(DocumentNominalLabelDTO doc) throws P em.persist(d); em.flush(); em.refresh(d); - System.out.println("Success in saving to document_nominal_label table, doc = " + doc.getIdDTO().getDocumentId()); + logger.info("Success in saving to document_nominal_label table, doc = " + doc.getIdDTO().getDocumentId()); } catch (Exception e) { - System.err.println("Error in saving document nominal label for document = " + doc.getIdDTO().getDocumentId()); - e.printStackTrace(); + logger.error("Error in saving document nominal label for document = " + doc.getIdDTO().getDocumentId(), e); tx.rollback(); return null; } @@ -83,10 +82,10 @@ public DocumentNominalLabelDTO addDocument(DocumentNominalLabelDTO doc) throws P em.merge(labeledDoc); em.flush(); if (!tx.wasCommitted()) tx.commit(); - System.out.println("Success in updating hashumanLabels field in document table, doc = " + labeledDoc.getDocumentId()); + logger.info("Success in updating hashumanLabels field in document table, doc = " + labeledDoc.getDocumentId()); return new DocumentNominalLabelDTO(d); } catch (Exception e) { - System.err.println("Error in updating hasHumanLabel field of labeled document = " + labeledDoc.getDocumentId() +", rolling back transaction (delete from document_nominal_label)..."); + logger.error("Error in updating hasHumanLabel field of labeled document = " + labeledDoc.getDocumentId() +", rolling back transaction (delete from document_nominal_label)..."); tx.rollback(); return null; } @@ -94,7 +93,6 @@ public DocumentNominalLabelDTO addDocument(DocumentNominalLabelDTO doc) throws P @Override public DocumentNominalLabelDTO editDocument(DocumentNominalLabelDTO doc) throws PropertyNotSetException { - System.out.println("Received request for: " + doc.getDocumentDTO().getDocumentID() + ", " + doc.getDocumentDTO().getCrisisDTO().getCode()); try { DocumentNominalLabel d = doc.toEntity(); DocumentNominalLabel oldDoc = getById(d.getId().getDocumentId()); @@ -105,8 +103,7 @@ public DocumentNominalLabelDTO editDocument(DocumentNominalLabelDTO doc) throws throw new RuntimeException("Not found"); } } catch (Exception e) { - System.out.println("Exception in merging/updating document: " + doc.getIdDTO().getDocumentId()); - e.printStackTrace(); + logger.error("Exception in merging/updating document: " + doc.getIdDTO().getDocumentId(), e); } return null; } @@ -117,6 +114,7 @@ public Integer deleteDocument(DocumentNominalLabelDTO doc) { DocumentNominalLabel managed = em.merge(doc.toEntity()); em.remove(managed); } catch (Exception e) { + logger.warn("Warning! Couldn't delete document nominal label with id : " + doc.getIdDTO()); return 0; } return 1; @@ -154,17 +152,16 @@ public boolean isDocumentExists(Long id) throws PropertyNotSetException { @Override public List getAllDocuments() throws PropertyNotSetException { - System.out.println("Received request for fetching all Documents!!!"); + logger.info("Received request for fetching all Documents!!!"); List dtoList = new ArrayList(); List list = getAll(); if (list != null && !list.isEmpty()) { for (DocumentNominalLabel doc : list) { - //System.out.println("Converting to DTO Document: " + doc.getDocumentId() + ", " + doc.getCrisis().getCode() + ", " + doc.isHasHumanLabels()); DocumentNominalLabelDTO dto = new DocumentNominalLabelDTO(doc); dtoList.add(dto); } } - System.out.println("Done creating DTO list, size = " + dtoList.size()); + logger.info("Done creating DTO list, size = " + dtoList.size()); return dtoList; } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentResourceFacadeImp.java index 4a768404f..32fc2a2dd 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/DocumentResourceFacadeImp.java @@ -165,7 +165,7 @@ public int deleteUnassignedDocument(Long documentID) { public int deleteUnassignedDocumentCollection(List documentIDList) { int deleteCount = 0; if (documentIDList != null && !documentIDList.isEmpty()) { - System.out.println("[deleteUnassignedDocumentCollection] Size of docList to delete: " + documentIDList.size()); + logger.info("[deleteUnassignedDocumentCollection] Size of docList to delete: " + documentIDList.size()); Session session = getCurrentSession(); try { Transaction tx = session.beginTransaction(); @@ -173,7 +173,6 @@ public int deleteUnassignedDocumentCollection(List documentIDList) { deleteCount += deleteUnassignedDocument(documentID); } tx.commit(); - System.out.println("[deleteUnassignedDocumentCollection] number of deleted records = " + deleteCount); } catch (Exception e) { logger.error("[deleteUnassignedDocumentCollection] Collection deletion query failed"); logger.error("Exception", e); @@ -192,8 +191,6 @@ public int deleteStaleDocuments(String joinType, String joinTable, String joinCo logger.info("received request: " + joinType + ", " + joinTable + ", " + joinColumn + ", " + maxTaskAge + ", " + scanInterval); - System.out.println("[deleteStaleDocuments] received request: " + joinType + ", " + joinTable + ", " - + joinColumn + ", " + maxTaskAge + ", " + scanInterval); int deleteCount = 0; Session session = getCurrentSession(); @@ -228,7 +225,6 @@ public int deleteStaleDocuments(String joinType, String joinTable, String joinCo Query deleteQuery = session.createSQLQuery(hql.toString()); deleteQuery.setParameter("task_expiry_age", Integer.parseInt(getTimeValue(maxTaskAge))); - System.out.println("Constructed query: " + deleteQuery.getQueryString()); logger.info("Constructed query: " + deleteQuery.getQueryString()); try { deleteCount = deleteQuery.executeUpdate(); @@ -303,13 +299,13 @@ public DocumentDTO addDocument(DocumentDTO doc) { em.refresh(d); return new DocumentDTO(d); } catch (Exception e) { + logger.error("Error in addDocument."); return null; } } @Override public DocumentDTO editDocument(DocumentDTO doc) throws PropertyNotSetException { - System.out.println("Received request for: " + doc.getDocumentID() + ", " + doc.getCrisisDTO().getCode()); try { Document d = doc.toEntity(); Document oldDoc = getById(d.getDocumentId()); @@ -320,8 +316,7 @@ public DocumentDTO editDocument(DocumentDTO doc) throws PropertyNotSetException throw new RuntimeException("Not found"); } } catch (Exception e) { - System.out.println("Exception in merging/updating document: " + doc.getDocumentID()); - e.printStackTrace(); + logger.error("Exception in merging/updating document: " + doc.getDocumentID(), e); } return null; } @@ -332,6 +327,7 @@ public Integer deleteDocument(DocumentDTO doc) { Document managed = em.merge(doc.toEntity()); em.remove(managed); } catch (Exception e) { + logger.warn("Warning! Couldn't delete document with ID : " + doc.getDocumentID()); return 0; } return 1; @@ -389,17 +385,15 @@ public boolean isDocumentExists(Long id) throws PropertyNotSetException { @Override public List getAllDocuments() throws PropertyNotSetException { - System.out.println("Received request for fetching all Documents!!!"); List dtoList = new ArrayList(); List list = getAll(); if (list != null && !list.isEmpty()) { for (Document doc : list) { - //System.out.println("Converting to DTO Document: " + doc.getDocumentId() + ", " + doc.getCrisis().getCode() + ", " + doc.isHasHumanLabels()); DocumentDTO dto = new DocumentDTO(doc); dtoList.add(dto); } } - System.out.println("Done creating DTO list, size = " + dtoList.size()); + logger.error("Done creating DTO list, size = " + dtoList.size()); return dtoList; } @@ -412,12 +406,11 @@ public List findLabeledDocumentsByCrisisID(Long crisisId) throws Pr List list = this.getAllByCriteria(criterion); if (list != null && !list.isEmpty()) { for (Document doc : list) { - //System.out.println("Converting to DTO Document: " + doc.getDocumentId() + ", " + doc.getCrisis().getCode() + ", " + doc.isHasHumanLabels()); DocumentDTO dto = new DocumentDTO(doc); dtoList.add(dto); } } - System.out.println("Done creating DTO list, size = " + dtoList.size()); + logger.info("Done creating DTO list, size = " + dtoList.size()); return dtoList; } @@ -430,12 +423,11 @@ public List findUnLabeledDocumentsByCrisisID(Long crisisId) throws List list = this.getAllByCriteria(criterion); if (list != null && !list.isEmpty()) { for (Document doc : list) { - //System.out.println("Converting to DTO Document: " + doc.getDocumentId() + ", " + doc.getCrisis().getCode() + ", " + doc.isHasHumanLabels()); DocumentDTO dto = new DocumentDTO(doc); dtoList.add(dto); } } - System.out.println("Done creating DTO list, size = " + dtoList.size()); + logger.info("Done creating DTO list, size = " + dtoList.size()); return dtoList; } @@ -449,21 +441,16 @@ public List getDocumentCollectionWithNominalLabelData(Long nominalL Criterion criterion = Restrictions.eq("hasHumanLabels", true); Criterion aliasCriterion = Restrictions.eq(aliasTableKeyField, nominalLabelID); - try { - List docList = this.getByCriteriaWithInnerJoinByOrder(criterion, "DESC", orderBy, null, aliasTable, aliasCriterion); - logger.debug("docList = " + docList); - if (docList != null && !docList.isEmpty()) { - logger.info("[getDocumentCollectionWithNominalLabelData] Fetched size = " + docList.size()); - NominalLabelDTO nominalLabel = nominalLabelEJB.getNominalLabelByID(nominalLabelID); - for (Document doc: docList) { - DocumentDTO dto = new DocumentDTO(doc); - dto.setNominalLabelDTO(nominalLabel); - dtoList.add(dto); - } - System.out.println("[getDocumentCollectionWithNominalLabelData] Done creating DTO list, size = " + dtoList.size()); + List docList = this.getByCriteriaWithInnerJoinByOrder(criterion, "DESC", orderBy, null, aliasTable, aliasCriterion); + if (docList != null && !docList.isEmpty()) { + logger.info("[getDocumentCollectionWithNominalLabelData] Fetched size = " + docList.size()); + NominalLabelDTO nominalLabel = nominalLabelEJB.getNominalLabelByID(nominalLabelID); + for (Document doc: docList) { + DocumentDTO dto = new DocumentDTO(doc); + dto.setNominalLabelDTO(nominalLabel); + dtoList.add(dto); } - } catch (Exception e) { - throw new Exception(); + logger.info("[getDocumentCollectionWithNominalLabelData] Done creating DTO list, size = " + dtoList.size()); } } return dtoList; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelFamilyResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelFamilyResourceFacadeImp.java index 031ee4738..2549bc7df 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelFamilyResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelFamilyResourceFacadeImp.java @@ -124,6 +124,7 @@ public List getTaggersByCodes(final List codes) { try { rows = query.getResultList(); } catch (NoResultException e) { + logger.warn("No result for codes : " + codes); return null; } for (Object[] row : rows) { diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelNominalLabelResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelNominalLabelResourceFacadeImp.java index 4a8967b15..77ac4db92 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelNominalLabelResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelNominalLabelResourceFacadeImp.java @@ -48,13 +48,14 @@ public List getAllModelNominalLabels() { try { List list = this.getAll(); if (list != null && !list.isEmpty()) { - System.out.println("Fetched list size: " + list.size()); + logger.info("Fetched list size: " + list.size()); for (ModelNominalLabel m : list) { dtoList.add(new ModelNominalLabelDTO(m)); } } return dtoList; } catch (Exception e) { + logger.error("Error in getAllModelNominalLabels."); return null; } } @@ -66,19 +67,18 @@ public List getAllModelNominalLabelsByModelID(Long modelID Model model = modelEJB.getById(modelID); if (model != null) { - System.out.println("Model is not NULL in getAllModelNominalLabelsByModelID for modelID = " + modelID + ",crisis code = " + crisisCode); + logger.info("Model is not NULL in getAllModelNominalLabelsByModelID for modelID = " + modelID + ",crisis code = " + crisisCode); Hibernate.initialize(model.getModelFamily()); try { Boolean modelStatus = model.getModelFamily().isIsActive(); //getting model status ModelFamily modelFamily = model.getModelFamily(); - System.out.println("*********** retrived modelFamilyID = " + modelFamily.getModelFamilyId()); Hibernate.initialize(modelFamily.getNominalAttribute()); Long nominalAttributeId = modelFamily.getNominalAttribute().getNominalAttributeId(); //Long nominalAttributeId = model.getModelFamily().getNominalAttribute().getNominalAttributeId(); modelNominalLabelList = this.getAllByCriteria(Restrictions.eq("id.modelId", modelID)); - System.out.println("modelNominalLabellist size = " + (modelNominalLabelList != null ? modelNominalLabelList.size() : "null")); + logger.info("modelNominalLabellist size = " + (modelNominalLabelList != null ? modelNominalLabelList.size() : "null")); for (ModelNominalLabel labelEntity : modelNominalLabelList) { @@ -95,12 +95,12 @@ public List getAllModelNominalLabelsByModelID(Long modelID if (document.getCrisisDTO().getCode().equals(crisisCode)) { ++trainingSet; } else { - System.out.println("Rejecting document for crisis = " + document.getCrisisDTO().getCode() + "having isEvaluationSet = " + document.getIsEvaluationSet() + ", nominalLabel = " + nominalLabel.getName() + ", model = " + model.getModelId() + " coming from modelFamily = " + modelFamily.getModelFamilyId()); + logger.info("Rejecting document for crisis = " + document.getCrisisDTO().getCode() + "having isEvaluationSet = " + document.getIsEvaluationSet() + ", nominalLabel = " + nominalLabel.getName() + ", model = " + model.getModelId() + " coming from modelFamily = " + modelFamily.getModelFamilyId()); } } - } catch (PropertyNotSetException e) { + } catch (Exception e) { // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getDocumentCollectionWithNominalLabelData.", e); } } @@ -115,8 +115,7 @@ public List getAllModelNominalLabelsByModelID(Long modelID dtoList.add(dto); } } catch (Exception e) { - System.out.println("Exception occured in getAllModelNominalLabelsByModelID \n\n" ); - e.printStackTrace(); + logger.error("Exception occured in getAllModelNominalLabelsByModelID \n\n", e); return null; } } @@ -137,14 +136,14 @@ public ModelNominalLabelDTO addModelNominalLabel(ModelNominalLabelDTO modelNomin em.refresh(modelLabel); return new ModelNominalLabelDTO(modelLabel); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error in addModelNominalLabel.", e); return null; } } @Override public ModelNominalLabelDTO editModelNominalLabel(ModelNominalLabelDTO modelNominalLabel) throws PropertyNotSetException { - System.out.println("Received request for: " + modelNominalLabel.getIdDTO().getNominalLabelId() + ":" + modelNominalLabel.getIdDTO().getModelId()); + logger.info("Received request for: " + modelNominalLabel.getIdDTO().getNominalLabelId() + ":" + modelNominalLabel.getIdDTO().getModelId()); try { ModelNominalLabel label = modelNominalLabel.toEntity(); ModelNominalLabel oldLabel = getByCriteria(Restrictions.eq("id", modelNominalLabel.getIdDTO().toEntity())); @@ -155,8 +154,7 @@ public ModelNominalLabelDTO editModelNominalLabel(ModelNominalLabelDTO modelNomi throw new RuntimeException("Not found"); } } catch (Exception e) { - System.out.println("Exception in merging/updating nominalLabel: " + modelNominalLabel.getIdDTO().getNominalLabelId()); - e.printStackTrace(); + logger.error("Exception in merging/updating nominalLabel: " + modelNominalLabel.getIdDTO().getNominalLabelId(), e); } return null; } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelResourceFacadeImp.java index e3738146b..10c0f83be 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/ModelResourceFacadeImp.java @@ -46,9 +46,8 @@ public ModelResourceFacadeImp() { public List getAllModels() throws PropertyNotSetException { List modelDTOList = new ArrayList(); List modelList = getAll(); - System.out.println("Fetched models list size: " + modelList.size()); + logger.info("Fetched models list size: " + modelList.size()); for (Model model : modelList) { - System.out.println("Adding model ID = " + model.getModelId()); modelDTOList.add(new ModelDTO(model)); } return modelDTOList; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/NominalAttributeResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/NominalAttributeResourceFacadeImp.java index 0d9fbc5c2..2594c24f1 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/NominalAttributeResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/NominalAttributeResourceFacadeImp.java @@ -40,7 +40,6 @@ public NominalAttributeDTO addAttribute(NominalAttributeDTO attribute) throws Pr try { NominalAttribute e = attribute.toEntity(); Long id = save(e); - System.out.println("Saved entity ID = " + id); if (id != null) { NominalAttributeDTO savedEntity = this.getAttributeByID(id); return savedEntity; @@ -140,10 +139,11 @@ public List getAllAttributesExceptCrisis(Long crisisID) thr attribute.setLabelName(((String) row[6])); attributesList.add(attribute); } - return attributesList; } catch (NoResultException e) { - return attributesList; + logger.warn("No result for crisisID : " + crisisID); } + + return attributesList; } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/NominalLabelResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/NominalLabelResourceFacadeImp.java index ea2a02527..856ca9e9c 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/NominalLabelResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/NominalLabelResourceFacadeImp.java @@ -44,14 +44,13 @@ public NominalLabelDTO addNominalLabel(NominalLabelDTO nominalLabel) { em.refresh(nb); return new NominalLabelDTO(nb); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error in addNominalLabel.", e); return null; } } @Override public NominalLabelDTO editNominalLabel(NominalLabelDTO nominalLabel) throws PropertyNotSetException { - System.out.println("Received request for: " + nominalLabel.getNominalLabelId() + ":" + nominalLabel.getNominalLabelCode()); try { NominalLabel label = nominalLabel.toEntity(); NominalLabel oldLabel = getById(label.getNominalLabelId()); @@ -62,8 +61,7 @@ public NominalLabelDTO editNominalLabel(NominalLabelDTO nominalLabel) throws Pro throw new RuntimeException("Not found"); } } catch (Exception e) { - System.out.println("Exception in merging/updating nominalLabel: " + nominalLabel.getNominalLabelId()); - e.printStackTrace(); + logger.error("Exception in merging/updating nominalLabel: " + nominalLabel.getNominalLabelId(), e); } return null; } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/SystemEventResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/SystemEventResourceFacadeImp.java index c75d8755c..0f852118a 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/SystemEventResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/SystemEventResourceFacadeImp.java @@ -2,6 +2,8 @@ import javax.ejb.Stateless; +import org.apache.log4j.Logger; + import qa.qcri.aidr.common.exception.PropertyNotSetException; import qa.qcri.aidr.dbmanager.dto.SystemEventDTO; import qa.qcri.aidr.dbmanager.ejb.local.facade.impl.CoreDBServiceFacadeImp; @@ -12,6 +14,7 @@ public class SystemEventResourceFacadeImp extends CoreDBServiceFacadeImp implements SystemEventResourceFacade { + private Logger logger = Logger.getLogger(SystemEventResourceFacadeImp.class); public SystemEventResourceFacadeImp() { super(SystemEvent.class); } @@ -24,12 +27,10 @@ public void insertSystemEvent(SystemEventDTO event) { em.flush(); em.refresh(sysevent); } catch (PropertyNotSetException e1) { - e1.printStackTrace(); + logger.error("Error in insertSystemEvent."); } - catch (Exception e) - { - System.out.println("Unable to save event to database"); - e.printStackTrace(); + catch (Exception e) { + logger.error("Unable to save event to database", e); } } @@ -52,10 +53,8 @@ public void insertSystemEvent(String severity, String module, String code, em.flush(); em.refresh(sysevent); } - catch (Exception e) - { - System.out.println("Unable to save event to database"); - e.printStackTrace(); + catch (Exception e){ + logger.error("Unable to save event to database", e); } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAnswerResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAnswerResourceFacadeImp.java index 243d6902a..767c29315 100755 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAnswerResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAnswerResourceFacadeImp.java @@ -41,8 +41,7 @@ public TaskAnswerDTO insertTaskAnswer(TaskAnswerDTO taskAnswer) { return getTaskAnswer(taskAnswer.getDocumentID(), taskAnswer.getUserID()); } catch (Exception e) { - System.out.println("Unable to save taskAnswer: " + taskAnswer.getDocumentID() + ", " + taskAnswer.getUserID() + ", " + taskAnswer.getAnswer()); - e.printStackTrace(); + logger.error("Unable to save taskAnswer: " + taskAnswer.getDocumentID() + ", " + taskAnswer.getUserID() + ", " + taskAnswer.getAnswer(), e); return null; } } else { diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAssignmentResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAssignmentResourceFacadeImp.java index c44aa07f2..ccb0b044d 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAssignmentResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/TaskAssignmentResourceFacadeImp.java @@ -37,7 +37,7 @@ public TaskAssignmentResourceFacadeImp() { @Override public int insertTaskAssignment(List taskList, Long userID) { // hard code, will create user service - System.out.println("[insertTaskAssignment] Going to insert/create task list of size = " + taskList.size() + ", for userID: " + userID); + logger.info("[insertTaskAssignment] Going to insert/create task list of size = " + taskList.size() + ", for userID: " + userID); try { for (Iterator it = taskList.iterator(); it.hasNext();){ DocumentDTO tb = (DocumentDTO) it.next(); @@ -61,7 +61,6 @@ public int insertOneTaskAssignment(Long documentID, Long userID) { try { List taskAssignments = getAllByCriteria(Restrictions.eq("id.documentId", documentID)); if(null == taskAssignments || taskAssignments.size()== 0){ - System.out.println("[insertOneTaskAssignment] Going to insert/create task list of size = " + taskAssignments.size() + ", for userID: " + userID); TaskAssignment taskAssignment = new TaskAssignment(documentID, userID, new Date()); save(taskAssignment); em.flush(); @@ -134,13 +133,11 @@ public int undoTaskAssignment(Long documentID, Long userID) { @Override public void undoTaskAssignmentByTimer() { try { - // System.out.println("undoTaskAssignmentByTimer is called"); DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); Calendar cal = Calendar.getInstance(); cal.add(Calendar.HOUR, -12); String calDate = dateFormat.format(cal.getTime()); List taskAssignments = getAllByCriteria(Restrictions.le("assignedAt",dateFormat.parse(calDate))); - // System.out.println("undoTaskAssignmentByTimer size : " + taskAssignments); if (taskAssignments != null) { for (Iterator it =taskAssignments.iterator(); it.hasNext();){ TaskAssignment taskAssignment = (TaskAssignment) it.next(); diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/UsersResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/UsersResourceFacadeImp.java index af64e7ff9..2f381faa3 100755 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/UsersResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/UsersResourceFacadeImp.java @@ -89,7 +89,7 @@ public UsersDTO addUser(UsersDTO user) { em.refresh(u); return new UsersDTO(u); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error in addUser.", e); return null; } } @@ -103,6 +103,7 @@ public Integer deleteUser(Long id) { return 1; } else { + logger.error("User requested to be deleted does not exist! id = " + id); throw new RuntimeException("User requested to be deleted does not exist! id = " + id); } } diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/entities/misc/SystemEvent.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/entities/misc/SystemEvent.java index 614cfcf52..42f7b0697 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/entities/misc/SystemEvent.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/entities/misc/SystemEvent.java @@ -12,6 +12,8 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; +import org.apache.log4j.Logger; + @Entity @Table(name = "system_event", catalog = "aidr_predict") public class SystemEvent implements java.io.Serializable{ @@ -20,6 +22,7 @@ public class SystemEvent implements java.io.Serializable{ * */ private static final long serialVersionUID = 6475156575527664114L; + private Logger logger = Logger.getLogger(SystemEvent.class); public SystemEvent(String severity, String module, String description) { this(severity,module,null,description); @@ -51,7 +54,7 @@ public SystemEvent(Date receivedAt, String severity, String module, String code, public SystemEvent(Date receivedAt, String severity, String module, String code, String description, Boolean emailSent) { super(); - System.out.println("date:" + receivedAt + " modeule:" + module +" emailSent?" + emailSent); + logger.info("date:" + receivedAt + " modeule:" + module +" emailSent?" + emailSent); this.receivedAt = receivedAt; this.severity = severity; this.module = module; From 9cd58ee8ca7ea980063e7d0c57559a276ba2a039 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Sat, 1 Aug 2015 15:31:37 +0530 Subject: [PATCH 33/44] Added logs to AIDR-TrainerAPI. --- .../trainer/api/Jedis/JedisDataStore.java | 2 +- .../aidr/trainer/api/Jedis/JedisNotifier.java | 13 +++---- .../controller/ClientAppSourceController.java | 2 +- .../api/controller/CrisisController.java | 11 ++++-- .../CustomUITemplateController.java | 8 ++--- .../api/controller/DocumentController.java | 6 ++-- .../trainer/api/controller/GeoController.java | 13 +++---- .../api/controller/TaskAnswerController.java | 2 +- .../api/controller/TemplateController.java | 7 +++- .../api/controller/TestTaskManager.java | 18 +++++----- .../trainer/api/dao/impl/AbstractDaoImpl.java | 14 ++++---- .../api/dao/impl/ClientAppSourceDaoImpl.java | 12 ++++--- .../api/dao/impl/TaskAssignmentDaoImpl.java | 22 +++++++----- .../api/entity/DocumentNominalLabel.java | 5 ++- .../aidr/trainer/api/entity/TaskAnswer.java | 7 ++-- .../impl/DocumentNominalLabelServiceImpl.java | 7 ++-- .../api/service/impl/DocumentServiceImpl.java | 19 +++++----- .../service/impl/TaskAnswerServiceImpl.java | 23 ++++-------- .../impl/TaskAssignmentServiceImpl.java | 10 +++--- .../api/service/impl/TemplateServiceImpl.java | 36 +++++++++++-------- .../api/template/CrisisJsonOutput.java | 8 +++-- .../trainer/api/template/PybossaTemplate.java | 9 +++-- .../aidr/trainer/api/util/Communicator.java | 12 +++---- .../qcri/aidr/trainer/api/util/GISUtil.java | 2 +- .../api/util/TaskManagerEntityMapper.java | 6 ++-- 25 files changed, 149 insertions(+), 125 deletions(-) diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisDataStore.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisDataStore.java index cc3e71010..5d0840779 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisDataStore.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisDataStore.java @@ -29,7 +29,7 @@ public static Jedis getJedisConnection() throws Exception { } return jedisPool.getResource(); } catch (Exception e) { - logger.error("Could not establish Redis connection. Is the Redis server running?"); + logger.error("Could not establish Redis connection. Is the Redis server running?",e); throw e; } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisNotifier.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisNotifier.java index 1bee42118..4498b14ef 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisNotifier.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/Jedis/JedisNotifier.java @@ -19,11 +19,10 @@ public class JedisNotifier { public JedisNotifier() { try { jedis = JedisDataStore.getJedisConnection(); - System.out.println("Obtained jedis connection: " + jedis); + //System.out.println("Obtained jedis connection: " + jedis); + logger.info("Redis connection established"); } catch (Exception e) { - logger.error("failed to create jedis connection"); - logger.error("exception", e); - e.printStackTrace(); + logger.error("failed to create jedis connection", e); } } @@ -33,14 +32,12 @@ public void notifyToJedis(String itemJosn){ try { // jedis.r logger.info("input data : " + itemJosn); - System.out.println("input data : " + itemJosn); + //System.out.println("input data : " + itemJosn); jedis.rpush("training_sample_info_stream", itemJosn); JedisDataStore.close(jedis); //outputCount++; } catch (Exception e) { - logger.error("Error when serializing output document."); - logger.error("exception", e); - e.printStackTrace(); + logger.error("Error when serializing output document.", e); } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppSourceController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppSourceController.java index c72a06ca0..e55017805 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppSourceController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/ClientAppSourceController.java @@ -57,7 +57,7 @@ public Response saveAppSource(String data){ } catch(Exception e){ returnValue = StatusCodeType.RETURN_FAIL; - logger.error("saveAppSource got exception : "); + logger.error("Exception while saving app source",e); } return Response.status(CodeLookUp.APP_REQUEST_SUCCESS).entity(returnValue).build(); diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CrisisController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CrisisController.java index 1aee86a89..b5c3430b9 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CrisisController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CrisisController.java @@ -1,9 +1,11 @@ package qa.qcri.aidr.trainer.api.controller; +import org.apache.log4j.Logger; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; + import qa.qcri.aidr.trainer.api.entity.Crisis; import qa.qcri.aidr.trainer.api.entity.NominalLabel; import qa.qcri.aidr.trainer.api.service.CrisisService; @@ -12,6 +14,7 @@ import javax.ws.rs.*; import javax.ws.rs.core.MediaType; + import java.util.List; import java.util.Set; @@ -25,6 +28,9 @@ @Path("/crisis") @Component public class CrisisController { + + protected static Logger logger = Logger.getLogger(CrisisController.class); + @Autowired private CrisisService crisisService; @@ -32,12 +38,11 @@ public class CrisisController { @Produces( MediaType.APPLICATION_JSON ) @Path("/id/{crisisid}") public CrisisJsonModel getCrisisByID(@PathParam("crisisid") Long crisisid){ - System.out.println("received request for crisisId = " + crisisid); + logger.info("received request for crisisId = " + crisisid); try { return crisisService.findByOptimizedCrisisID(crisisid); } catch (Exception e) { - System.out.println("Error in getting crisisID for crisis ID = " + crisisid); - e.printStackTrace(); + logger.error("Error in getting crisisID for crisis ID = " + crisisid,e); } return null; } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CustomUITemplateController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CustomUITemplateController.java index cf6c27d8a..bb957a02a 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CustomUITemplateController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/CustomUITemplateController.java @@ -47,7 +47,7 @@ public List getLandingUIByID(@PathParam("crisisID") Long crisi @Produces( MediaType.APPLICATION_JSON ) @Path("/get/customUI/{crisisID}") public List getCustomUIByID(@PathParam("crisisID") Long crisisID){ - System.out.println("[getCustomUIByID] Received request for crisisID = " + crisisID); + logger.info("[getCustomUIByID] Received request for crisisID = " + crisisID); return customUITemplateService.getCustomTemplateForLandingPage(crisisID); } @@ -82,7 +82,7 @@ public void updateWelcomePage(String data){ } catch(Exception e){ - logger.debug("updateWelcomePage. Exception: " + data); + logger.debug("Exception while updating welcome page " + data,e); } } @@ -115,7 +115,7 @@ public void updateTutorial(String data){ } catch(Exception e){ - logger.debug("updateTutorial. Exception: " + data); + logger.debug("Exception while updating tutorial " + data,e); } //updateCustomTemplateByAttribute(Long crisisID, Long attributeID, int customUIType, int skinType) @@ -150,7 +150,7 @@ public void updateSkin(String data){ } catch(Exception e){ - logger.debug("updateTutorial. Exception: " + data); + logger.debug("Exception while updating skin " + data,e); } //updateCustomTemplateByAttribute(Long crisisID, Long attributeID, int customUIType, int skinType) diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/DocumentController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/DocumentController.java index e80152b56..f38fd9903 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/DocumentController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/DocumentController.java @@ -53,20 +53,20 @@ public List getOneTaskBufferToAssign(@PathParam("crisisID") DocumentDTO document = null; Long id = new Long(crisisID); - System.out.println("Going to fetch internal training document for crisisID = " + crisisID + ", userName = " + userName + ", count = " + maxresult); + logger.info("Going to fetch internal training document for crisisID = " + crisisID + ", userName = " + userName + ", count = " + maxresult); if(userName != null){ List documents = documentService.getDocumentForOneTask(id,Integer.valueOf(maxresult),userName ); if(documents!= null){ if(documents.size() > 0){ document = documents.get(0); - System.out.println("Fetched document for internal tagging task: " + document.getDocumentID() + ", for crisisID = " + document.getCrisisDTO().getCrisisID()); + logger.info("Fetched document for internal tagging task: " + document.getDocumentID() + ", for crisisID = " + document.getCrisisDTO().getCrisisID()); } } } List jsonData = documentService.findOneDocumentForTaskByCririsID(document, id); for (int i = 0;i < jsonData.size();i++) { - System.out.println("To be returned json Data, documentID = " + jsonData.get(i).getDocumentID() + ", for crisisID = " + jsonData.get(i).getCrisisID() ); + logger.info("To be returned json Data, documentID = " + jsonData.get(i).getDocumentID() + ", for crisisID = " + jsonData.get(i).getCrisisID() ); } return jsonData; } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/GeoController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/GeoController.java index 91a97b265..8ad8eaced 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/GeoController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/GeoController.java @@ -50,6 +50,7 @@ public String getMapGeoJSONBasedOnDate(@PathParam("lastupdated") String lastupda queryDate = sdf.parse(lastupdated); } catch(Exception e){ + logger.error("Exception while parsing date in getMapGeoJSONBasedOnDate",e); queryDate = null; } } @@ -58,8 +59,7 @@ public String getMapGeoJSONBasedOnDate(@PathParam("lastupdated") String lastupda returnValue = geoService.getGeoJsonOuputJSON(queryDate); } catch (Exception e) { - logger.error("Exception getMapGeoJSONBasedOnDate"); - // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + logger.error("Exception while gettting MapGeoJSON Based On Date \t"+e.getStackTrace()); } return returnValue; } @@ -75,8 +75,7 @@ public String getMapGeoJSON() { returnValue = geoService.getGeoJsonOuputJSON(null); } catch (Exception e) { - logger.error("Exception getMapGeoJSON"); - // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + logger.error("Exception while getting MapGeoJSON \t"+e.getStackTrace()); } return returnValue; } @@ -99,8 +98,7 @@ public String getMapGeoJSONPBasedOnDate(@PathParam("lastupdated") String lastupd returnValue = geoService.getGeoJsonOuputJSONP(queryDate); } catch (Exception e) { - logger.error("Exception getMapGeoJSONPBasedOnDate"); - //e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + logger.error("Exception while getting MapGeoJSONP BasedOnDate \t"+e.getStackTrace()); } return returnValue; @@ -117,8 +115,7 @@ public String getMapGeoJSONP() { returnValue = geoService.getGeoJsonOuputJSONP(null); } catch (Exception e) { - logger.error("Exception getMapGeoJSONP"); - // e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + logger.error("Exception while getting MapGeoJSONP \t"+e.getStackTrace()); } return returnValue; diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAnswerController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAnswerController.java index 26ca5b34e..2fa1cbb49 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAnswerController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TaskAnswerController.java @@ -40,7 +40,7 @@ public void saveTaskAnswer(String data){ taskAnswerService.processTaskAnswer(data); } catch(Exception e){ - logger.error("saveTaskAnswer got exception on: " + data); + logger.error("Error while saving Task Answer",e); } // below is non transaction block diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TemplateController.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TemplateController.java index 622a14202..e2c78b3e4 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TemplateController.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TemplateController.java @@ -1,7 +1,9 @@ package qa.qcri.aidr.trainer.api.controller; +import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; + import qa.qcri.aidr.trainer.api.service.TemplateService; import qa.qcri.aidr.trainer.api.template.CrisisApplicationListModel; import qa.qcri.aidr.trainer.api.template.CrisisLandingHtmlModel; @@ -12,6 +14,7 @@ import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; + import java.util.List; /** @@ -24,6 +27,8 @@ @Path("/template") @Component public class TemplateController { + + protected static Logger logger = Logger.getLogger(TemplateController.class); @Autowired private TemplateService templateService; @@ -64,7 +69,7 @@ public CrisisLandingHtmlModel getCrisisTemplateByID(@PathParam("crisisid") Long @Produces( MediaType.APPLICATION_JSON ) @Path("/status/crisis/code/{code}") public CrisisLandingStatusModel getCrisisTemplateByID(@PathParam("code") String code){ - System.out.println("[getCrisisTemplateByID] received request for code = " + code); + logger.info("[getCrisisTemplateByID] received request for code = " + code); return templateService.getCrisisLandingStatusByCrisisCode(code); } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TestTaskManager.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TestTaskManager.java index 589e4b6fe..e9a748c21 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TestTaskManager.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/controller/TestTaskManager.java @@ -1,16 +1,15 @@ package qa.qcri.aidr.trainer.api.controller; -import javax.ws.rs.core.Context; -import javax.naming.InitialContext; -import javax.naming.NamingException; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; +import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -21,6 +20,8 @@ @Path("/test") @Component public class TestTaskManager { + + protected static Logger logger = Logger.getLogger(TestTaskManager.class); @Context private UriInfo context; @@ -52,9 +53,9 @@ public Response test() { taskManager = (TaskManagerRemote) ctx.lookup(remoteEJBJNDIName); */ - System.out.println("taskManager: " + taskManager + ", time taken to initialize = " + (System.currentTimeMillis() - startTime)); + //System.out.println("taskManager: " + taskManager + ", time taken to initialize = " + (System.currentTimeMillis() - startTime)); if (taskManager != null) { - System.out.println("Success in connecting to remote EJB to initialize taskManager"); + logger.info("Success in connecting to remote EJB to initialize taskManager"); } long elapsed = 0L; @@ -67,17 +68,16 @@ public Response test() { } else { respString.append("null"); } - System.out.println("[main] " + respString.toString() + ", time taken = " + elapsed); + logger.info("[main] " + respString.toString() + ", time taken = " + elapsed); } catch (Exception e) { - System.err.println("Error in JNDI lookup"); + logger.error("Error in JNDI lookup",e); respString.append("Error in JNDI lookup"); - e.printStackTrace(); } return Response.ok(respString.toString()).build(); } public static void main(String[] args) throws Exception { TestTaskManager tc = new TestTaskManager(); - System.out.println("Result: " + tc.test().toString()); + logger.info("Result: " + tc.test().toString()); } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/AbstractDaoImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/AbstractDaoImpl.java index 44c0c7e77..5bc8b8e3f 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/AbstractDaoImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/AbstractDaoImpl.java @@ -67,7 +67,7 @@ public List findAll() { //System.out.println("result = " + result); return result; } catch (HibernateException e) { - logger.error("Error in findAll()."); + logger.error("Error in findAll().",e); return null; } @@ -88,7 +88,7 @@ public List findByCriteria(Criterion criterion) { try { return criteria.list(); } catch (HibernateException e) { - logger.error("Error in findByCriteria for criteria : " + criteria.toString()); + logger.error("Error in findByCriteria for criteria : " + criteria.toString(),e); return null; } } @@ -103,7 +103,7 @@ public List getMaxOrderByCriteria(Criterion criterion, String orderBy) { try { return criteria.list(); } catch (HibernateException e) { - logger.error("Error in getMaxOrderByCriteria for criteria : " + criteria.toString()); + logger.error("Error in getMaxOrderByCriteria for criteria : " + criteria.toString(),e); return null; } } @@ -122,7 +122,7 @@ public List findByCriteriaByOrder(Criterion criterion, String[] orderBy, Inte try { return criteria.list(); } catch (HibernateException e) { - logger.error("Error in findbyCriteriaByOrder for criteria : " + criteria.toString()); + logger.error("Error in findbyCriteriaByOrder for criteria : " + criteria.toString(),e); return null; } } @@ -143,7 +143,7 @@ public List findByCriteriaWithAliasByOrder(Criterion criterion, String[] orde try { return criteria.list(); } catch (HibernateException e) { - logger.error("Error in findByCriteriaWithAliasByOrder for criteria : " + criteria.toString()); + logger.error("Error in findByCriteriaWithAliasByOrder for criteria : " + criteria.toString(),e); return null; } } @@ -160,7 +160,7 @@ public List findByCriteria(Criterion criterion, Integer count) { try { return criteria.list(); } catch (HibernateException e) { - logger.error("Error in findByCriteria for criteria : " + criteria.toString()); + logger.error("Error in findByCriteria for criteria : " + criteria.toString(),e); return null; } } @@ -173,7 +173,7 @@ public E findByCriterionID(Criterion criterion) { try { return (E) criteria.uniqueResult(); } catch (HibernateException e) { - logger.error("Error in findByCriterionID for criteria : " + criteria.toString()); + logger.error("Error in findByCriterionID for criteria : " + criteria.toString(),e); return null; } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/ClientAppSourceDaoImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/ClientAppSourceDaoImpl.java index c2360b1a8..76bfff82b 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/ClientAppSourceDaoImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/ClientAppSourceDaoImpl.java @@ -1,13 +1,15 @@ package qa.qcri.aidr.trainer.api.dao.impl; +import java.util.List; + +import org.apache.log4j.Logger; import org.hibernate.criterion.Restrictions; import org.springframework.stereotype.Repository; + import qa.qcri.aidr.trainer.api.dao.ClientAppSourceDao; import qa.qcri.aidr.trainer.api.entity.ClientAppSource; import qa.qcri.aidr.trainer.api.store.StatusCodeType; -import java.util.List; - /** * Created with IntelliJ IDEA. * User: jlucas @@ -17,6 +19,8 @@ */ @Repository public class ClientAppSourceDaoImpl extends AbstractDaoImpl implements ClientAppSourceDao { + + private static Logger logger=Logger.getLogger(ClientAppSourceDaoImpl.class); protected ClientAppSourceDaoImpl(){ super(ClientAppSource.class); @@ -48,12 +52,12 @@ public void acceptSource(String fileURL, Long clientAppID) { List sources = findActiveSourcePerClient( clientAppID ); if(sources.size() > 0){ - System.out.println("sources : EXTERNAL_DATA_SOURCE_UPLOADED"); + logger.info("sources : EXTERNAL_DATA_SOURCE_UPLOADED"); ClientAppSource ca1 = new ClientAppSource(clientAppID, StatusCodeType.EXTERNAL_DATA_SOURCE_UPLOADED, fileURL); save(ca1); } else{ - System.out.println("sources : EXTERNAL_DATA_SOURCE_ACTIVE"); + logger.info("sources : EXTERNAL_DATA_SOURCE_ACTIVE"); ClientAppSource ca2 = new ClientAppSource(clientAppID, StatusCodeType.EXTERNAL_DATA_SOURCE_ACTIVE, fileURL); save(ca2); } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/TaskAssignmentDaoImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/TaskAssignmentDaoImpl.java index 7d09a7c4f..ad6343a75 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/TaskAssignmentDaoImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/dao/impl/TaskAssignmentDaoImpl.java @@ -1,18 +1,26 @@ package qa.qcri.aidr.trainer.api.dao.impl; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; import org.hibernate.criterion.Restrictions; import org.springframework.stereotype.Repository; + import qa.qcri.aidr.trainer.api.dao.TaskAssignmentDao; import qa.qcri.aidr.trainer.api.entity.Document; import qa.qcri.aidr.trainer.api.entity.TaskAssignment; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; - @Repository public class TaskAssignmentDaoImpl extends AbstractDaoImpl implements TaskAssignmentDao{ + + private static Logger logger=Logger.getLogger(TaskAssignmentDaoImpl.class); protected TaskAssignmentDaoImpl(){ super(TaskAssignment.class); @@ -30,8 +38,6 @@ public void insertTaskAssignment(List documents, Long userID) { save(taskAssignment); } } - - } @Override @@ -88,7 +94,7 @@ public void undoTaskAssignmentByTimer() { } } catch (Exception e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + logger.error("Exception while undoTaskAssignmentByTimer \t"+e.getStackTrace()); } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/entity/DocumentNominalLabel.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/entity/DocumentNominalLabel.java index 9e5b634db..1bf684d89 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/entity/DocumentNominalLabel.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/entity/DocumentNominalLabel.java @@ -3,6 +3,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import java.io.Serializable; @@ -19,6 +20,8 @@ @XmlRootElement @JsonIgnoreProperties(ignoreUnknown=true) public class DocumentNominalLabel implements Serializable { + + private static Logger logger=Logger.getLogger(DocumentNominalLabel.class); private static final long serialVersionUID = -5527566248002296042L; @@ -68,7 +71,7 @@ public static DocumentNominalLabel toLocalDocumentNominalLabel(DocumentNominalLa DocumentNominalLabel nominalDoc = new DocumentNominalLabel(doc.getIdDTO().getDocumentId(), doc.getIdDTO().getNominalLabelId(), doc.getIdDTO().getUserId()); return nominalDoc; } catch (PropertyNotSetException e) { - e.printStackTrace(); + logger.error("Exception while parsing DocumentNominalLabelDTO to LocalDocumentNominalLabel",e); } } return null; diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/entity/TaskAnswer.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/entity/TaskAnswer.java index b9968bce0..315ad8794 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/entity/TaskAnswer.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/entity/TaskAnswer.java @@ -3,6 +3,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.log4j.Logger; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import qa.qcri.aidr.dbmanager.dto.TaskAnswerDTO; @@ -17,6 +18,8 @@ @XmlRootElement @JsonIgnoreProperties(ignoreUnknown=true) public class TaskAnswer implements Serializable { + + private static Logger logger=Logger.getLogger(TaskAnswer.class); private static final long serialVersionUID = -5527566248002296042L; @@ -89,7 +92,7 @@ public TaskAnswer toLocalTaskAnswer(TaskAnswerDTO t) { return null; } } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while parsing TaskAnswerDTO to LocalTaskAnswer",e); return null; } } @@ -100,7 +103,7 @@ public TaskAnswerDTO toDTO() { dto.setFromTrustedUser(this.isFromTrustedUser()); return dto; } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while parsing TaskAnswer to TaskAnswerDTO",e); return null; } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentNominalLabelServiceImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentNominalLabelServiceImpl.java index 6a4254b29..a3a2ae6ce 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentNominalLabelServiceImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentNominalLabelServiceImpl.java @@ -1,5 +1,6 @@ package qa.qcri.aidr.trainer.api.service.impl; +import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; @@ -27,6 +28,8 @@ @Service("documentNominalLabelService") @Transactional(readOnly = true, propagation = Propagation.REQUIRED) public class DocumentNominalLabelServiceImpl implements DocumentNominalLabelService { + + private static Logger logger=Logger.getLogger(DocumentNominalLabelServiceImpl.class); //@Autowired //private DocumentNominalLabelDao documentNominalLabelDao; @@ -50,7 +53,7 @@ public void saveDocumentNominalLabel(DocumentNominalLabel documentNominalLabel) DocumentNominalLabelDTO dto = new DocumentNominalLabelDTO(id, nb, doc); taskManager.saveDocumentNominalLabel(dto); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while saving documentNominalLabel",e); } } @@ -61,7 +64,7 @@ public boolean foundDuplicateEntry(DocumentNominalLabel documentNominalLabel) { DocumentNominalLabelIdDTO id = new DocumentNominalLabelIdDTO(documentNominalLabel.getDocumentID(), documentNominalLabel.getNominalLabelID(), documentNominalLabel.getUserID()); return remoteDocumentNominalLabelEJB.isDocumentExists(id); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while finding Duplicate Entry for documentNominalLabel",e); return false; } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentServiceImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentServiceImpl.java index 31ea29e34..875e7bbb8 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentServiceImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/DocumentServiceImpl.java @@ -1,6 +1,12 @@ package qa.qcri.aidr.trainer.api.service.impl; //import org.apache.log4j.Logger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -13,8 +19,6 @@ import qa.qcri.aidr.task.common.TrainingDataFetchType; import qa.qcri.aidr.task.ejb.TaskManagerRemote; import qa.qcri.aidr.trainer.api.entity.Crisis; - - //import qa.qcri.aidr.trainer.api.entity.Document; import qa.qcri.aidr.trainer.api.entity.Users; import qa.qcri.aidr.trainer.api.service.CrisisService; @@ -27,12 +31,6 @@ import qa.qcri.aidr.trainer.api.template.NominalAttributeJsonModel; import qa.qcri.aidr.trainer.api.template.TaskBufferJsonModel; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - /** * Created with IntelliJ IDEA. * User: jilucas @@ -70,8 +68,7 @@ public void updateHasHumanLabel(Long documentID, boolean value) { DocumentDTO dto = (DocumentDTO) taskManager.setTaskParameter(Document.class, documentID, paramMap); //logger.info("Update of hasHumanLabels successful for document " + dto.getDocumentID() + ", crisisId = " + dto.getCrisisDTO().getCrisisID()); } catch (Exception e) { - logger.error("Update unsuccessful for documentID = " + documentID); - logger.error("Exception", e); + logger.error("Update unsuccessful for documentID = " + documentID, e); } } @@ -170,7 +167,7 @@ private List getJsonModeForTask(long crisisID, DocumentDTO public List getAvailableDocument(Long crisisID, Integer maxresult){ //return documentDao.findDocumentForTask(crisisID, maxresult); List dtoList = taskManager.getNewTaskCollection(crisisID, maxresult, "DESC", null); - System.out.println("Fetched from DB manager, documents list size = " + dtoList.size()); + logger.info("Fetched from DB manager, documents list size = " + dtoList.size()); return dtoList; } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java index e75663016..d8b01a01a 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAnswerServiceImpl.java @@ -9,6 +9,8 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import qa.qcri.aidr.dbmanager.dto.DocumentDTO; +import qa.qcri.aidr.dbmanager.dto.TaskAnswerDTO; import qa.qcri.aidr.task.ejb.TaskManagerRemote; import qa.qcri.aidr.trainer.api.Jedis.JedisNotifier; import qa.qcri.aidr.trainer.api.entity.DocumentNominalLabel; @@ -21,9 +23,6 @@ import qa.qcri.aidr.trainer.api.template.PybossaTemplate; import qa.qcri.aidr.trainer.api.template.TaskAnswerResponse; -import qa.qcri.aidr.dbmanager.dto.DocumentDTO; -import qa.qcri.aidr.dbmanager.dto.TaskAnswerDTO; - /** * Created with IntelliJ IDEA. * User: jilucas, koushik @@ -78,9 +77,7 @@ public void pushTaskAnswerToJedis(TaskAnswerResponse taskAnswerResponse) { //logger.info("jedisNotifier created : " + jedisNotifier); } catch (Exception e){ - logger.error("jedisNotifier creation error for :" + taskAnswerResponse.getDocumentID()); - logger.error("exception", e); - e.printStackTrace(); + logger.error("jedisNotifier creation error for :" + taskAnswerResponse.getDocumentID() + e.getStackTrace()); } } @@ -115,8 +112,7 @@ public void addToTaskAnswer(TaskAnswerResponse taskAnswerResponse){ try { taskManager.insertTaskAnswer(t); } catch (Exception e) { - logger.error("[addToTaskAnswer] Error in saving task answer : " + taskAnswer); - e.printStackTrace(); + logger.error("[addToTaskAnswer] Error in saving task answer : " + taskAnswer+"\t"+e.getStackTrace()); } } @@ -165,9 +161,8 @@ public void processTaskAnswer(String data) { try { taskManager.insertTaskAnswer(t); } catch (Exception e) { - logger.error("exception", e); + logger.error("Error in processing task answer:"+e.getStackTrace()); //System.err.println("[processTaskAnswer] Error in saving task answer : " + taskAnswer); - e.printStackTrace(); } } @@ -190,9 +185,7 @@ public void processTaskAnswer(String data) { logger.info("jedisNotifier created : " + jedisNotifier); } catch (Exception e){ - logger.error("jedisNotifier creation error for: " + data); - logger.error("exception", e); - e.printStackTrace(); + logger.error("jedisNotifier creation error for: " + data+"\t"+e.getStackTrace()); } } //System.out.println("Attempting to push to JEDIS now: " + taskAnswerResponse.getDocumentID()); @@ -208,9 +201,7 @@ public void processTaskAnswer(String data) { } } catch(Exception e){ - logger.error("Exception for saving task answer data : " + data); - logger.error("exception", e); - e.printStackTrace(); + logger.error("Exception for saving task answer data : " + data+"\t"+e.getStackTrace()); } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAssignmentServiceImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAssignmentServiceImpl.java index 529aee81f..0c9b84363 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAssignmentServiceImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TaskAssignmentServiceImpl.java @@ -48,8 +48,7 @@ public void revertTaskAssignmentByUserName(Long documentID, String userName) { try { taskManager.undoTaskAssignment(documentID, userID); } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error(" Error in revertTaskAssignmentByUserName for User: "+userName+"\t"+e.getStackTrace()); } } } @@ -63,8 +62,7 @@ public void revertTaskAssignment(Long documentID, Long userID) { taskManager.undoTaskAssignment(documentID, userID); //logger.info("Removed from taskAssignment table: documentID = " + documentID + ", userID = " + userID); } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error(" Error while reverting Task Assignment for userID: "+userID+"\t"+e.getStackTrace()); } } @@ -86,7 +84,7 @@ public void addToTaskAssignment(List documents, Long userID){ //System.out.println("[addToTaskAssignment] Going to insert task list of size = " + documents.size() + ", for userID: " + userID); taskManager.assignNewTaskToUser(documents, userID); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error while adding Task Assignment for userID="+userID+"\t"+e.getStackTrace()); } } @@ -99,7 +97,7 @@ public void addToOneTaskAssignment(Long documentID, Long userID){ taskManager.assignNewTaskToUser(documentID, userID); } catch (Exception e) { // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error while adding To OneTaskAssignment for user ID="+userID+"\t"+e.getStackTrace()); } } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TemplateServiceImpl.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TemplateServiceImpl.java index c4b9f28dd..33096a258 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TemplateServiceImpl.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/service/impl/TemplateServiceImpl.java @@ -1,19 +1,26 @@ package qa.qcri.aidr.trainer.api.service.impl; +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.Logger; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import qa.qcri.aidr.common.exception.PropertyNotSetException; import qa.qcri.aidr.dbmanager.dto.CrisisDTO; import qa.qcri.aidr.dbmanager.ejb.remote.facade.CrisisResourceFacade; import qa.qcri.aidr.trainer.api.entity.Client; import qa.qcri.aidr.trainer.api.entity.ClientApp; //import qa.qcri.aidr.trainer.api.entity.Crisis; import qa.qcri.aidr.trainer.api.entity.CustomUITemplate; -import qa.qcri.aidr.trainer.api.service.*; +import qa.qcri.aidr.trainer.api.service.ClientAppService; +import qa.qcri.aidr.trainer.api.service.ClientService; +import qa.qcri.aidr.trainer.api.service.CustomUITemplateService; +import qa.qcri.aidr.trainer.api.service.TaskQueueService; +import qa.qcri.aidr.trainer.api.service.TemplateService; import qa.qcri.aidr.trainer.api.store.CodeLookUp; import qa.qcri.aidr.trainer.api.store.StatusCodeType; import qa.qcri.aidr.trainer.api.store.URLReference; @@ -22,9 +29,6 @@ import qa.qcri.aidr.trainer.api.template.CrisisLandingHtmlModel; import qa.qcri.aidr.trainer.api.template.CrisisLandingStatusModel; -import java.util.ArrayList; -import java.util.List; - /** * Created with IntelliJ IDEA. * User: jlucas @@ -36,6 +40,8 @@ @Service("templateService") @Transactional(readOnly = false) public class TemplateServiceImpl implements TemplateService { + + private static Logger logger=Logger.getLogger(TemplateServiceImpl.class); @Autowired private ClientService clientService; @@ -110,7 +116,7 @@ public List getApplicationListHtmlByCrisisCode(Strin } } } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception in getApplicationListHtmlByCrisisCode, crisisCode="+crisisCode+"\t"+e.getStackTrace()); return null; } return crisisApplicationListModelList; //To change body of implemented methods use File | Settings | File Templates. @@ -123,8 +129,7 @@ public CrisisLandingHtmlModel getCrisisLandingHtmlByCrisisCode(String crisisCode try { crisisList = crisisService.findByCriteria("code", crisisCode); } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Exception in CrisisLandingHtmlModel for crisis="+crisisCode+"\t"+e.getStackTrace()); } if(crisisList != null){ if(crisisList.size() > 0){ @@ -154,7 +159,7 @@ public CrisisLandingHtmlModel getCrisisLandingHtmlByCrisisID(Long crisisID) { } } } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception in getCrisisLandingHtmlByCrisisID for crisisID="+crisisID+"\t"+e.getStackTrace()); } return crisisLandingHtmlModel; } @@ -168,7 +173,7 @@ public String getCrisisLandingJSONPByCrisisID(Long crisisID) { //Crisis crisis = crisisService.findByCrisisID(crisisID); crisis = crisisService.findCrisisByID(crisisID); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while finding crisis by id:"+crisisID ,e); } List clientAppList = clientAppService.getAllClientAppByCrisisID(crisisID); // customUITemplateService @@ -236,7 +241,7 @@ public String getCrisisLandingJSONPByCrisisCode(String crisisCode) { } } } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception in getCrisisLandingJSONPByCrisisCode for crisis="+crisisCode+"\t"+e.getStackTrace()); } return returnValue; //To change body of implemented methods use File | Settings | File Templates. } @@ -247,16 +252,17 @@ public CrisisLandingStatusModel getCrisisLandingStatusByCrisisCode(String crisis List crisisList = null; try { crisisList = crisisService.findByCriteria("code", crisisCode); - System.out.println("crisisList size = " + (crisisList != null ? crisisList.size() : "null")); + logger.info("crisisList size = " + (crisisList != null ? crisisList.size() : "null")); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception in getCrisisLandingStatusByCrisisCode for crisis: "+crisisCode+"\t"+e.getStackTrace()); } boolean isReadyToShow = false; if(crisisList != null){ if(crisisList.size() > 0){ CrisisDTO crisis = crisisList.get(0); List clientAppList = clientAppService.getAllClientAppByCrisisID(crisis.getCrisisID()); - System.out.println("clientAppList size = " + (clientAppList != null ? clientAppList.size() : "null")); + //System.out.println("clientAppList size = " + (clientAppList != null ? clientAppList.size() : "null")); + logger.info("clientAppList size = " + (clientAppList != null ? clientAppList.size() : "null")); if(clientAppList != null ){ if(clientAppList.size() > 0){ isReadyToShow = true; @@ -264,7 +270,7 @@ public CrisisLandingStatusModel getCrisisLandingStatusByCrisisCode(String crisis } } } - System.out.println("isReadyToShow = " + isReadyToShow); + logger.info("isReadyToShow = " + isReadyToShow); if(isReadyToShow){ String url = URLReference.PUBLIC_LINK + crisisCode; crisisLandingStatusModel = new CrisisLandingStatusModel(url, StatusCodeType.CRISIS_PYBOSSA_SERVICE_READY, "ready" ); diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/template/CrisisJsonOutput.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/template/CrisisJsonOutput.java index 640b81a45..d9fa551f8 100755 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/template/CrisisJsonOutput.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/template/CrisisJsonOutput.java @@ -8,6 +8,8 @@ import java.util.HashSet; import java.util.Set; +import org.apache.log4j.Logger; + /** * Created with IntelliJ IDEA. * User: jilucas @@ -16,13 +18,15 @@ * To change this template use File | Settings | File Templates. */ public class CrisisJsonOutput { + + private static Logger logger=Logger.getLogger(CrisisJsonOutput.class); public CrisisJsonModel crisisJsonModelGenerator(Crisis crisis){ CrisisJsonModel crisisJsonModel = new CrisisJsonModel(); if (crisis != null) { - System.out.println("received crisis = " + crisis.getCode() + ", id = " + crisis.getCrisisID()); + logger.info("received crisis = " + crisis.getCode() + ", id = " + crisis.getCrisisID()); } else { - System.out.println("received crisis = " + crisis); + logger.info("received crisis = " + crisis); } crisisJsonModel.setCrisisID(crisis.getCrisisID()); crisisJsonModel.setCode(crisis.getCode()); diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/template/PybossaTemplate.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/template/PybossaTemplate.java index e2f0b5995..7bf08b58b 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/template/PybossaTemplate.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/template/PybossaTemplate.java @@ -1,9 +1,11 @@ package qa.qcri.aidr.trainer.api.template; +import org.apache.log4j.Logger; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; + import qa.qcri.aidr.trainer.api.entity.*; import qa.qcri.aidr.trainer.api.service.CrisisService; @@ -18,6 +20,8 @@ * To change this template use File | Settings | File Templates. */ public class PybossaTemplate { + + private static Logger logger=Logger.getLogger(PybossaTemplate.class); public TaskAnswerResponse getPybossaTaskAnswer(String data, CrisisService crisisService){ @@ -56,7 +60,7 @@ public TaskAnswerResponse getPybossaTaskAnswer(String data, CrisisService crisis ModelFamily currentModelFamily = modelFamily; Set nominalLabelSet = modelFamily.getNominalAttribute().getNominalLabelSet(); - System.out.print("attribute name : " + modelFamily.getNominalAttribute().getName() + "\n"); + logger.info("attribute name : " + modelFamily.getNominalAttribute().getName() + "\n"); for (NominalLabel nominalLabel : nominalLabelSet){ JSONObject taskAnswerElement = new JSONObject(); if(nominalLabel.getNominalAttributeID().equals(attributeID)) { @@ -91,7 +95,8 @@ public TaskAnswerResponse getPybossaTaskAnswer(String data, CrisisService crisis } } catch (ParseException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + //e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + logger.error("Exception while getting Pybossa task answer \t"+e.getStackTrace()); } return taskAnswerResponse; } diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/Communicator.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/Communicator.java index af074bf4a..0825817fa 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/Communicator.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/Communicator.java @@ -64,7 +64,7 @@ public int sendPut(String data, String url) { } }catch (Exception ex) { - logger.warn("Error in processing request for data : " + data + " and url : " + url); + logger.warn("Error in processing request for data : " + data + " and url : " + url,ex); } finally { httpClient.getConnectionManager().shutdown(); } @@ -103,7 +103,7 @@ public String deleteGet(String url){ }catch (Exception ex) { - logger.error("ex Code deleteGet2: " + url); + logger.error("Exception in deleteGet2: " + url,ex); } finally { httpClient.getConnectionManager().shutdown(); } @@ -151,8 +151,8 @@ public int sendPost(String data, String url) { }catch (Exception ex) { - logger.error("ex Code sendPost2: " + data); - logger.error("ex Code sendPost3: " + url); + logger.error("Exception in sendPost2: " + data,ex); + logger.error("Exception in sendPost3: " + url,ex); } finally { httpClient.getConnectionManager().shutdown(); } @@ -195,7 +195,7 @@ public String sendPostGet(String data, String url) { }catch (Exception ex) { - logger.error("Error in processing request for data : " + data + " and url :" + url); + logger.error("Error in processing request for data : " + data + " and url :" + url,ex); responseOutput.append("Exception Code : " + ex); } finally { @@ -231,7 +231,7 @@ public String sendGet(String url) { in.close(); }catch (Exception ex) { - logger.error("sendGet url = " + url); + logger.error("sendGet url = " + url,ex); } return response.toString(); diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/GISUtil.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/GISUtil.java index 6167367c7..a3cf25e9f 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/GISUtil.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/GISUtil.java @@ -82,7 +82,7 @@ public String getDisplayNameWithReverseLookUp(String key) { } } catch(Exception e){ - logger.error("getDisplayNameWithReverseLookUp exception for key: " + key); + logger.error("getDisplayNameWithReverseLookUp exception for key: " + key,e); } } return info; diff --git a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/TaskManagerEntityMapper.java b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/TaskManagerEntityMapper.java index 6e8fa032a..2f5441ffc 100644 --- a/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/TaskManagerEntityMapper.java +++ b/aidr-trainer-api/src/main/java/qa/qcri/aidr/trainer/api/util/TaskManagerEntityMapper.java @@ -23,7 +23,7 @@ public E deSerializeList(String jsonString, TypeReference type) { return docList; } } catch (Exception e) { - logger.error("JSON deserialization exception"); + logger.error("JSON deserialization exception",e); } return null; } @@ -40,7 +40,7 @@ public E deSerialize(String jsonString, Class entityType) { return entity; } } catch (Exception e) { - logger.error("JSON deserialization exception"); + logger.error("JSON deserialization exception",e); } return null; } @@ -53,7 +53,7 @@ public String serializeTask(E task) { try { if (task != null) jsonString = mapper.writeValueAsString(task); } catch (IOException e) { - logger.error("JSON serialization exception"); + logger.error("JSON serialization exception",e); } return jsonString; } From be7a249540f1a8a83037686d42ecae0299306798 Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Mon, 3 Aug 2015 12:02:15 +0530 Subject: [PATCH 34/44] Commit for #97809356 : aidr-tagger-api logging --- .../predictui/api/CollectionResource.java | 3 - .../aidr/predictui/api/CrisisResource.java | 28 +++++----- .../predictui/api/CrisisTypeResource.java | 9 +++ .../api/CustomUITemplateResource.java | 1 - .../aidr/predictui/api/DocumentResource.java | 2 +- .../qcri/aidr/predictui/api/MiscResource.java | 18 +++--- .../predictui/api/ModelFamilyResource.java | 11 +--- .../api/ModelNominalLabelResource.java | 9 +-- .../aidr/predictui/api/TaskBufferScanner.java | 9 ++- .../predictui/api/TestDBManagerResource.java | 52 ++++-------------- .../qcri/aidr/predictui/api/UserResource.java | 3 +- .../facade/imp/CollectionResourceImp.java | 5 ++ .../CrisisManagementResourceFacadeImp.java | 3 - .../facade/imp/CrisisTypeResourceImp.java | 34 +++--------- .../facade/imp/CustomUITemplateFacadeImp.java | 15 ++++- .../facade/imp/DocumentFacadeImp.java | 5 +- .../facade/imp/MiscResourceFacadeImp.java | 55 ++++++++----------- .../predictui/facade/imp/ModelFacadeImp.java | 2 +- .../facade/imp/NominalLabelResourceImp.java | 25 +++------ .../facade/imp/SystemEventFacadeImp.java | 6 +- .../predictui/facade/imp/UserResourceImp.java | 1 - 21 files changed, 121 insertions(+), 175 deletions(-) diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CollectionResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CollectionResource.java index c608607ad..a98b4d6c7 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CollectionResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CollectionResource.java @@ -50,9 +50,6 @@ public Response getAllCrisisByUserID(@PathParam("userID") int userID) { } response.setCollections(collections); return Response.ok(response).build(); - - - } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisResource.java index 70b4c9335..22d238b21 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisResource.java @@ -68,6 +68,7 @@ public Response getCrisisByID(@PathParam("id") Long id) { crisis = crisisLocalEJB.getCrisisByID(id); System.out.println("fetched crisis for id " + id + ": " + (crisis != null ? crisis.getCode() : "null")); } catch (RuntimeException e) { + logger.error("Error in getCrisisById for crisis id : " + id); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } ObjectMapper mapper = new ObjectMapper(); @@ -75,7 +76,7 @@ public Response getCrisisByID(@PathParam("id") Long id) { return Response.ok(mapper.writeValueAsString(crisis)).build(); } catch (JsonProcessingException e) { // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in Json processing."); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } } @@ -88,7 +89,7 @@ public Response getCrisisByCode(@PathParam("code") String crisisCode) { ResponseWrapper response = new ResponseWrapper(); try { crisis = crisisLocalEJB.getCrisisByCode(crisisCode); - System.out.println("For code = " + crisisCode + ", Returning crisis: " + (crisis != null ? crisis.getCode() : "null")); + logger.info("For code = " + crisisCode + ", Returning crisis: " + (crisis != null ? crisis.getCode() : "null")); if (crisis != null) { response.setDataObject(crisis); response.setTotal(1); @@ -105,6 +106,7 @@ public Response getCrisisByCode(@PathParam("code") String crisisCode) { response.setTotal(0); response.setStatusCode(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED)); response.setMessage(e.getCause().getCause().getMessage()); + logger.error("Error in getCrisisByCode for code : " + crisisCode); return Response.ok(response).build(); } @@ -155,7 +157,6 @@ public Response getCrisesByCodes(List codes) { @Produces(MediaType.APPLICATION_JSON) @Path("/all") public Response getAllCrisis() { - System.out.println("Received request"); List crisisList = crisisLocalEJB.getAllCrisis(); ResponseWrapper response = new ResponseWrapper(); @@ -163,13 +164,11 @@ public Response getAllCrisis() { response.setCrisises(crisisList); ObjectMapper mapper = new ObjectMapper(); - System.out.println("retreived crisis list: " + crisisList); + logger.info("retreived crisis list: " + crisisList); try { - System.out.println("Serialized response: " + mapper.writeValueAsString(response)); - //return Response.ok(response).build(); return Response.ok(mapper.writeValueAsString(response)).build(); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error in getAllCrisis."); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } } @@ -178,7 +177,7 @@ public Response getAllCrisis() { @Produces(MediaType.APPLICATION_JSON) public ResponseWrapper getAllCrisisByUserID(@QueryParam("userID") Long userID) throws Exception { List crisisList = crisisLocalEJB.getAllCrisisByUserID(userID); - System.out.println("list of crisis for userID " + userID + ": " + (crisisList != null ? crisisList.size() : 0)); + logger.info("list of crisis for userID " + userID + ": " + (crisisList != null ? crisisList.size() : 0)); ResponseWrapper response = new ResponseWrapper(); if (crisisList == null) { response.setMessage("No crisis found associated with the given user id."); @@ -193,7 +192,7 @@ public ResponseWrapper getAllCrisisByUserID(@QueryParam("userID") Long userID) t @Produces(MediaType.APPLICATION_JSON) public Response addCrisis(CrisisDTO crisis) { try { - System.out.println("Received request to add crisis: " + crisis.getCode()); + logger.info("Received request to add crisis: " + crisis.getCode()); if (crisisLocalEJB.isCrisisExists(crisis.getCode())) { logger.warn("Crisis with code = " + crisis.getCode() + " already has at least one classifier attached!"); return Response.ok(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)).build(); @@ -201,7 +200,7 @@ public Response addCrisis(CrisisDTO crisis) { // Otherwise, add the new crisis to aidr_predict database crisis.setIsTrashed(false); CrisisDTO newCrisis = crisisLocalEJB.addCrisis(crisis); - System.out.println("Added crisis successfully: id = " + newCrisis.getCrisisID() + ", " + newCrisis.getCode()); + logger.info("Added crisis successfully: id = " + newCrisis.getCrisisID() + ", " + newCrisis.getCode()); return Response.ok(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)).build(); } catch (RuntimeException e) { logger.error("Error while adding Crisis. Possible causes could be duplication of primary key, incomplete data, incompatible data format. For crisis: " + crisis.getCode(), e); @@ -214,10 +213,11 @@ public Response addCrisis(CrisisDTO crisis) { @Produces(MediaType.APPLICATION_JSON) public Response editCrisis(CrisisDTO crisis) { try { - System.out.println("Received request to edit crisis: " + crisis.getCode()); + logger.info("Received request to edit crisis: " + crisis.getCode()); CrisisDTO updatedCrisis = crisisLocalEJB.editCrisis(crisis); return Response.ok(updatedCrisis).build(); } catch (RuntimeException e) { + logger.error("Error in editCrisis for crisis code : " + crisis.getCode()); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } @@ -234,18 +234,16 @@ public Response getNominalAttributesCountForCrisis(@PathParam("crisisCode") Stri List crisisList = new ArrayList(); crisisList.add(crisisCode); Map retVal = crisisLocalEJB.countClassifiersByCrisisCodes(crisisList); - System.out.println("retrieved result: " + retVal); if (retVal != null) { result.put("count", retVal.get(crisisCode) != null ? retVal.get(crisisCode) : 0); - System.out.println("Generated response: " + result); + logger.info("Generated response: " + result); return Response.ok(mapper.writeValueAsString(result)).build(); } } result.put("count", 0); - System.out.println("Response string from tagger-api on classifier count: " + mapper.writeValueAsString(result)); return Response.ok(mapper.writeValueAsString(result)).build(); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error in getNominalAttributesCountForCrisis for crisis code : " + crisisCode); result.put("count", -1); return Response.ok(result).build(); } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisTypeResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisTypeResource.java index 8fc01d7e9..56722a7be 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisTypeResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CrisisTypeResource.java @@ -28,6 +28,9 @@ import javax.ws.rs.core.Response; + +import org.apache.log4j.Logger; + //import qa.qcri.aidr.predictui.dto.CrisisTypeDTO; //import qa.qcri.aidr.predictui.entities.CrisisType; import qa.qcri.aidr.predictui.facade.CrisisTypeResourceFacade; @@ -41,6 +44,8 @@ @Stateless public class CrisisTypeResource { + private Logger logger = Logger.getLogger(CrisisTypeResource.class); + @Context private UriInfo context; @EJB @@ -57,6 +62,7 @@ public Response getCrisisTypeByID(@PathParam("id") Long id) { try { crisistype = crisisTypeLocal.getCrisisTypeByID(id); } catch (RuntimeException e) { + logger.error("Error in getCrisisTypeByID for id : " + id); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } return Response.ok(crisistype).build(); @@ -79,6 +85,7 @@ public Response addCrisisType(CrisisTypeDTO crisis) { try { crisis = crisisTypeLocal.addCrisisType(crisis); } catch (RuntimeException e) { + logger.error("Error in addCrisisType."); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } return Response.ok(crisis).build(); @@ -91,6 +98,7 @@ public Response editCrisisType(CrisisTypeDTO crisis) { try { crisis = crisisTypeLocal.editCrisisType(crisis); } catch (RuntimeException e) { + logger.error("Error in editCrisisType."); return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), e.getCause().getCause().getMessage())).build(); } return Response.ok(crisis).build(); @@ -103,6 +111,7 @@ public Response deleteCrisisType(@PathParam("id") Long id) { try { crisisTypeLocal.deleteCrisisType(id); } catch (RuntimeException e) { + logger.error("Error while deleting CrisisType. Possible cause(s): (1) Given crisis-type ID does not exist."); return Response.ok( new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), "Error while deleting CrisisType. Possible cause(s): (1) Given crisis-type ID does not exist.")).build(); diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CustomUITemplateResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CustomUITemplateResource.java index 229aef4ad..199ea6034 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CustomUITemplateResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/CustomUITemplateResource.java @@ -49,7 +49,6 @@ public CustomUITemplateResource(){ @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public Response addNewTemplate(CustomUiTemplateDTO customUITemplate) { - System.out.println("*************************************************************"); boolean isUpdate = false; CustomUiTemplateDTO dbTemplate = null; int type = customUITemplate.getTemplateType(); diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/DocumentResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/DocumentResource.java index cd04a4585..408b4cb02 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/DocumentResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/DocumentResource.java @@ -129,7 +129,7 @@ public Response getNominalAttributesCountForCrisis(@PathParam("crisisId") Long c } return Response.ok(mapper.writeValueAsString(unlabeledCount)).build(); } catch (Exception e) { - e.printStackTrace(); + logger.error("Error in getNominalAttributesCountForCrisis for crisis Id : " + crisisId); return Response.ok(-1).build(); } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java index 7c5599060..92807e2f2 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java @@ -116,7 +116,6 @@ public Response getHumanLabeledDocumentsByCrisisID(@PathParam("crisisID") Long c } try { List dtoList = miscEJB.getHumanLabeledDocumentsByCrisisID(crisisID, count); - System.out.println("REST call will return dto List size = " + (dtoList != null ? dtoList.size() : "null")); if (dtoList != null) { ResponseWrapper response = new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)); response.setItems(dtoList); @@ -126,7 +125,7 @@ public Response getHumanLabeledDocumentsByCrisisID(@PathParam("crisisID") Long c return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS), "Found 0 human labeled documents")).build(); } } catch (Exception e) { - logger.error("Exception", e); + logger.error("Error in getHumanLabeledDocumentsByCrisisID for crisisId : " + crisisID, e); return Response.ok( new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), "Exception in fetching human labeled documents")).build(); } @@ -152,7 +151,7 @@ public Response getHumanLabeledDocumentsByCrisisCode(@PathParam("crisisCode") St return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS), "Found 0 human labeled documents")).build(); } } catch (Exception e) { - logger.error("Exception", e); + logger.error("Error in getHumanLabeledDocumentsByCrisisCode for code : " + crisisCode, e); return Response.ok( new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), "Exception in fetching human labeled documents")).build(); } @@ -169,7 +168,6 @@ public Response getHumanLabeledDocumentsByCrisisIDUserID(@PathParam("crisisID") } try { List dtoList = miscEJB.getHumanLabeledDocumentsByCrisisIDUserID(crisisID, userID, count); - System.out.println("REST call will return dto List size = " + (dtoList != null ? dtoList.size() : "null")); if (dtoList != null) { ResponseWrapper response = new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)); response.setItems(dtoList); @@ -179,7 +177,8 @@ public Response getHumanLabeledDocumentsByCrisisIDUserID(@PathParam("crisisID") return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS), "Found 0 human labeled documents")).build(); } } catch (Exception e) { - logger.error("Exception", e); + logger.error("Error in getHumanLabeledDocumentsByCrisisIDUserID for crisisID :" + + userID + " and userID : " + userID, e); return Response.ok( new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), "Exception in fetching human labeled documents")).build(); } @@ -198,7 +197,6 @@ public Response getHumanLabeledDocumentsByCrisisIDUserName(@PathParam("crisisID" } try { List dtoList = miscEJB.getHumanLabeledDocumentsByCrisisIDUserName(crisisID, userName, count); - System.out.println("REST call will return dto List size = " + (dtoList != null ? dtoList.size() : "null")); if (dtoList != null) { ResponseWrapper response = new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)); response.setItems(dtoList); @@ -208,7 +206,8 @@ public Response getHumanLabeledDocumentsByCrisisIDUserName(@PathParam("crisisID" return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS), "Found 0 human labeled documents")).build(); } } catch (Exception e) { - logger.error("Exception", e); + logger.error("Error in getHumanLabeledDocumentsByCrisisIDUserName for crisisID : " + + crisisID + " and userName : " + userName, e); return Response.ok( new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), "Exception in fetching human labeled documents")).build(); } @@ -235,7 +234,6 @@ public Response downloadHumanLabeledDocumentsByCrisisIDUserName(String queryStri } try { List dtoList = miscEJB.getHumanLabeledDocumentsByCrisisCode(crisisCode, count); - logger.info("REST call will return dto List size = " + (dtoList != null ? dtoList.size() : "null")); if(count==-1){ count=null; } @@ -251,8 +249,7 @@ public Response downloadHumanLabeledDocumentsByCrisisIDUserName(String queryStri return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS), "Found 0 human labeled documents")).build(); } } catch (Exception e) { - logger.error("Exception", e); - e.printStackTrace(); + logger.error("Error in downloadHumanLabeledDocumentsByCrisisIDUserName.", e); return Response.ok( new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), "Exception in fetching human labeled documents")).build(); } @@ -279,6 +276,7 @@ public Response sendErrorEmail(@FormParam("code") String code, @FormParam("modul systemEventEJB.insertSystemEvent("ERROR", module, code, description, emailSent); } catch (Exception e) { + logger.warn("Error in inserting system event."); return Response.serverError().build(); } return Response.ok().build(); diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelFamilyResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelFamilyResource.java index f73154347..6bac4b1ba 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelFamilyResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelFamilyResource.java @@ -91,21 +91,15 @@ public Response getModelByID(@PathParam("id") Long id) { public Response addCrisisAttribute(ModelFamilyDTO modelFamilyDTO) { // because ModelFamily has @XmlTransient annotation for crises and crisis was always null ResponseWrapper response = new ResponseWrapper(); - System.out.println("Received request to add attribute = " + modelFamilyDTO.getNominalAttributeDTO().getNominalAttributeId() + " for crisis = " + modelFamilyDTO.getCrisisDTO().getCrisisID()); + logger.info("Received request to add attribute = " + modelFamilyDTO.getNominalAttributeDTO().getNominalAttributeId() + " for crisis = " + modelFamilyDTO.getCrisisDTO().getCrisisID()); try { boolean retval = modelFamilyLocalEJB.addCrisisAttribute(modelFamilyDTO); - System.out.println("addedCrisisAttribute = " + retval); if (retval) { // TODO: move the following code to db-manager and return modelFamilyID directly from db-manager List dtoList = modelFamilyLocalEJB.getAllModelFamiliesByCrisis(modelFamilyDTO.getCrisisDTO().getCrisisID()); - if (dtoList != null) { - System.out.println("fetched list size for crisisID " + modelFamilyDTO.getCrisisDTO().getCrisisID() + " = " + dtoList.size()); - } for (ModelFamilyDTO mf : dtoList) { - System.out.println("Looking at: crisisID = " + mf.getCrisisDTO().getCrisisID() + ", " + mf.getNominalAttributeDTO().getNominalAttributeId()); if (mf.getCrisisDTO().getCrisisID().equals(modelFamilyDTO.getCrisisDTO().getCrisisID()) && mf.getNominalAttributeDTO().getNominalAttributeId().equals(modelFamilyDTO.getNominalAttributeDTO().getNominalAttributeId())) { - System.out.println("Found added modelFamily = " + mf.getModelFamilyId() + " for crisis = " + modelFamilyDTO.getCrisisDTO().getCrisisID()); response.setStatusCode(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)); response.setMessage("Adding Attribute to crisis " + modelFamilyDTO.getCrisisDTO().getCrisisID() + " succeeded"); response.setEntityID(mf.getModelFamilyId()); @@ -118,8 +112,7 @@ public Response addCrisisAttribute(ModelFamilyDTO modelFamilyDTO) { response.setEntityID(new Long(-1)); return Response.ok(response).build(); } catch (RuntimeException e) { - System.out.println("Error while adding Crisis attribute. Possible causes could be duplication of primary key, incomplete data, incompatible data format: " + modelFamilyDTO.getCrisisDTO().getCode() + "," + modelFamilyDTO.getNominalAttributeDTO().getCode()); - logger.error("Exception", e); + logger.error("Error while adding Crisis attribute. Possible causes could be duplication of primary key, incomplete data, incompatible data format: " + modelFamilyDTO.getCrisisDTO().getCode() + "," + modelFamilyDTO.getNominalAttributeDTO().getCode(), e); response.setStatusCode(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED)); response.setMessage("Adding Attribute to crisis " + modelFamilyDTO.getCrisisDTO().getCrisisID() + " failed due to exception" + e); response.setEntityID(new Long(-1)); diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelNominalLabelResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelNominalLabelResource.java index 1c91a6661..155a40acc 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelNominalLabelResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/ModelNominalLabelResource.java @@ -57,12 +57,9 @@ public Response getAllModelNominalLabelsByModelID(@PathParam("modelID") Long mod List modelNominalLabelList = modelNominalLabel.getAllModelNominalLabelsByModelID(modelID, code); ResponseWrapper response = new ResponseWrapper(); if (modelNominalLabelList != null){ - response.setModelNominalLabelsDTO(modelNominalLabelList); - for (ModelNominalLabelDTO dto: modelNominalLabelList) { - System.out.println("[getAllModelNominalLabelsByModelID] Training count for label: " + dto.getNominalLabelDTO().getName() + " is = " + dto.getTrainingDocuments()); - } - return Response.ok(response).build(); - } + response.setModelNominalLabelsDTO(modelNominalLabelList); + return Response.ok(response).build(); + } response.setMessage("no labels found for the given model-id."); return Response.ok(response).build(); } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TaskBufferScanner.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TaskBufferScanner.java index 1e26b0ef1..b1fe74250 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TaskBufferScanner.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TaskBufferScanner.java @@ -89,7 +89,7 @@ public boolean isThreadRunning() { } public boolean startTaskBufferScannerThread(final String taskMaxAge, final String taskScanInterval) { - System.out.println("[startTaskBufferScannerThread] Attempting to start new thread"); + logger.info("[startTaskBufferScannerThread] Attempting to start new thread"); scan = true; try { executorService.execute(new Runnable() { @@ -191,7 +191,6 @@ public Response stopScanner() { @Path("/ping") @Produces("application/json") public Response pingTaskBufferScanner() { - //System.out.println("[pingTaskBufferScanner] Received PING request"); String responseStr = null; if (isThreadRunning()) { responseStr = "{\"application\":\"TaskBufferScanner\", \"status\":\"RUNNING\"}"; @@ -214,12 +213,16 @@ void shutdownAndAwaitTermination() { logger.error("[shutdownAndAwaitTermination] Pool did not terminate"); } } catch (InterruptedException ie) { + + logger.warn("Warning! unable to clean up."); // (Re-)Cancel if current thread also interrupted executorService.shutdownNow(); // Preserve interrupt status try { Thread.currentThread().interrupt(); - } catch (Exception e) {} + } catch (Exception e) { + logger.warn("Warning! interrupt."); + } } executorService = null; } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TestDBManagerResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TestDBManagerResource.java index 76411a896..afacaa160 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TestDBManagerResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TestDBManagerResource.java @@ -52,7 +52,7 @@ public class TestDBManagerResource { public Response getCrisisTypeByID(@PathParam("id") long id) { try { CrisisTypeDTO dto = remoteCrisisTypeEJB.findCrisisTypeByID(id); - System.out.println("Returned successfully from remote EJB call!"); + logger.info("Returned successfully from remote EJB call!"); if (dto != null) { return Response.ok(dto).build(); @@ -61,7 +61,6 @@ public Response getCrisisTypeByID(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching crisisType from remote EJB: " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned").build(); @@ -73,7 +72,6 @@ public Response getCrisisTypeByID(@PathParam("id") long id) { public Response getAllCrisisTypes() { try { List dtoList = remoteCrisisTypeEJB.getAllCrisisTypes(); - System.out.println("Returned successfully from remote EJB call!"); if (dtoList != null) { return Response.ok(dtoList).build(); @@ -82,7 +80,6 @@ public Response getAllCrisisTypes() { // TODO Auto-generated catch block logger.error("Exception in fetching crisisType list from remote EJB "); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned").build(); @@ -94,7 +91,6 @@ public Response getAllCrisisTypes() { public Response getAllCrisisForCrisisType(@PathParam("id") long id) { try { List dtoList = remoteCrisisTypeEJB.getAllCrisisForCrisisTypeID(id); - System.out.println("Returned successfully from remote EJB call!"); if (dtoList != null) { return Response.ok(dtoList).build(); @@ -103,7 +99,6 @@ public Response getAllCrisisForCrisisType(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching crisis list for crisisType from remote EJB: " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned").build(); @@ -124,7 +119,6 @@ public Response getCrisisByID(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching crisis from remote EJB for id = " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned for crisis ID = " + id).build(); @@ -144,7 +138,6 @@ public Response getCrisisWithAllParamsByID(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching crisis from remote EJB for id = " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned for crisis ID = " + id).build(); @@ -156,7 +149,6 @@ public Response getCrisisWithAllParamsByID(@PathParam("id") long id) { public Response getAllCrisis() { try { List dtoList = remoteCrisisEJB.getAllCrisis(); - System.out.println("Returned successfully from remote EJB call!"); if (dtoList != null) { return Response.ok(dtoList).build(); @@ -165,7 +157,6 @@ public Response getAllCrisis() { // TODO Auto-generated catch block logger.error("Exception in fetching crisis from remote EJB"); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned").build(); @@ -179,7 +170,7 @@ public Response updateCrisisByID(@PathParam("id") long id, @QueryParam("value") CrisisDTO dto = remoteCrisisEJB.findCrisisByID(id); if (dto != null) { - System.out.println("Fetched crisis: " + dto.getCrisisID() + ", " + dto.getCrisisID()); + logger.info("Fetched crisis: " + dto.getCrisisID() + ", " + dto.getCrisisID()); dto.setIsTrashed(value); CrisisDTO newDTO = remoteCrisisEJB.editCrisis(dto); return Response.ok(newDTO).build(); @@ -187,7 +178,6 @@ public Response updateCrisisByID(@PathParam("id") long id, @QueryParam("value") } catch (Exception e) { logger.error("Exception in updating crisis from remote EJB: " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned for: " + id).build(); @@ -207,7 +197,6 @@ public Response getDocumentByID(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching document from remote EJB for id = " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned for document ID = " + id).build(); @@ -227,7 +216,6 @@ public Response getDocumentsByCrisisID(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching document from remote EJB for id = " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned for document ID = " + id).build(); @@ -247,7 +235,6 @@ public Response getLabeledDocumentsByCrisisID(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching document from remote EJB for id = " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned for document ID = " + id).build(); @@ -267,7 +254,6 @@ public Response getWithAllFieldsDocument(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching document from remote EJB for id = " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned for document ID = " + id).build(); @@ -287,7 +273,6 @@ public Response getUserByID(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching user from remote EJB for id = " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned for document ID = " + id).build(); @@ -308,7 +293,6 @@ public Response getDocumentNominalLabelByID(@PathParam("id") long id) { // TODO Auto-generated catch block logger.error("Exception in fetching document_nominal_label from remote EJB for id = " + id); logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("Null object returned for document ID = " + id).build(); @@ -323,13 +307,12 @@ public Response TestAddDeleteCrisisType() { CrisisTypeDTO dto = remoteCrisisTypeEJB.addCrisisType(crisisType); if (dto != null) { - System.out.println("Add crisis successful: " + dto.getName() + ":" + dto.getCrisisTypeId()); + logger.info("Add crisis successful: " + dto.getName() + ":" + dto.getCrisisTypeId()); Integer ret = remoteCrisisTypeEJB.deleteCrisisType(dto.getCrisisTypeId()); if (ret != null && ret.intValue() == 1) return Response.ok("CrisisType Add-Delete test successful" + dto).build(); } } catch (Exception e) { - logger.error("stacktrace: ", e); - e.printStackTrace(); + logger.error("Error in /crisisType/addde.", e); return Response.ok("Exception: " + e).build(); } return Response.ok("CrisisType add-Delete test failed").build(); @@ -347,14 +330,13 @@ public Response TestAddDeleteCrisis() { CrisisDTO dto = remoteCrisisEJB.addCrisis(newCrisis); if (dto != null) { - System.out.println("Add crisis successful: " + dto.getCrisisID() + ":" + dto.getName() + ":" + dto.getCode() + logger.info("Add crisis successful: " + dto.getCrisisID() + ":" + dto.getName() + ":" + dto.getCode() + ":" + dto.getCrisisTypeDTO().getName() + ":" + dto.getCrisisTypeDTO().getCrisisTypeId()); Integer ret = remoteCrisisEJB.deleteCrisis(dto); if (ret != null && ret.intValue() == 1) return Response.ok("Crisis Add-Delete test successful" + dto).build(); } } catch (Exception e) { - logger.error("stacktrace: ", e); - e.printStackTrace(); + logger.error("Error in /crisis/adddel.", e); return Response.ok("Exception: " + e).build(); } return Response.ok("Crisis add-Delete test failed").build(); @@ -370,14 +352,13 @@ public Response TestAddDeleteDocument() { DocumentDTO dto = remoteDocumentEJB.addDocument(newDoc); if (dto != null) { - System.out.println("Add document successful: " + dto.getDocumentID() + ":" + dto.getCrisisDTO().getCode() + logger.info("Add document successful: " + dto.getDocumentID() + ":" + dto.getCrisisDTO().getCode() + ":" + dto.getCrisisDTO().getCrisisID()); Integer ret = remoteDocumentEJB.deleteDocument(dto); if (ret != null && ret.intValue() == 1) return Response.ok("Document Add-Delete test successful" + dto).build(); } } catch (Exception e) { - logger.error("stacktrace: ", e); - e.printStackTrace(); + logger.error("Error in /document/adddel." e); return Response.ok("Exception: " + e).build(); } return Response.ok("Document add-Delete test failed").build(); @@ -392,13 +373,12 @@ public Response TestAddDeleteUser() { UsersDTO dto = remoteUsersEJB.addUser(newUser); if (dto != null) { - System.out.println("Add user successful: " + dto.getUserID() + ":" + dto.getName() + ":" + dto.getRole()); + logger.info("Add user successful: " + dto.getUserID() + ":" + dto.getName() + ":" + dto.getRole()); Integer ret = remoteUsersEJB.deleteUser(dto.getUserID()); if (ret != null && ret.intValue() == 1) return Response.ok("User Add-Delete test successful" + dto).build(); } } catch (Exception e) { logger.error("stacktrace: ", e); - e.printStackTrace(); return Response.ok("Exception: " + e).build(); } return Response.ok("User add-Delete test failed").build(); @@ -410,12 +390,6 @@ public Response TestAddDeleteUser() { public Response TestGetNewTask(@PathParam("crisisID") long crisisID) { System.out.println("Going to fetch new task collection for crisis ID = " + crisisID); try { - DocumentDTO dto = taskManager.getNewTask(crisisID, null); - if (dto != null) { - System.out.println("Fetched DTO: " + dto.getDocumentID() + dto.getCrisisDTO().getCrisisID()); - } else { - System.out.println("Fetched DTO: null"); - } List dtoList = taskManager.getNewTaskCollection(crisisID, 10, "DESC", null); if (dtoList != null) { return Response.ok(dtoList).build(); @@ -423,8 +397,7 @@ public Response TestGetNewTask(@PathParam("crisisID") long crisisID) { return Response.ok("Null object returned for crisis ID = " + crisisID).build(); } } catch (Exception e) { - logger.error("stacktrace: ", e); - e.printStackTrace(); + logger.error("Error in TestGetNewTask.", e); return Response.ok("Exception: " + e).build(); } } @@ -433,7 +406,7 @@ public Response TestGetNewTask(@PathParam("crisisID") long crisisID) { @Produces(MediaType.APPLICATION_JSON) @Path("/document/nominalLabel/{nominalLabelID}") public Response TestGetLabeled(@PathParam("nominalLabelID") long nominalLabelID) { - System.out.println("Going to fetch labled documents for nominalLabel ID = " + nominalLabelID); + logger.info("Going to fetch labled documents for nominalLabel ID = " + nominalLabelID); try { List dtoList = taskManager.getNominalLabelDocumentCollection(nominalLabelID); if (dtoList != null) { @@ -442,8 +415,7 @@ public Response TestGetLabeled(@PathParam("nominalLabelID") long nominalLabelID) return Response.ok("Null object returned for nominalLabelID = " + nominalLabelID).build(); } } catch (Exception e) { - logger.error("stacktrace: ", e); - e.printStackTrace(); + logger.error("Error in TestGetLabeled for label id : " + nominalLabelID, e); return Response.ok("Exception: " + e).build(); } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/UserResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/UserResource.java index 262fc1904..8bf818cb9 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/UserResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/UserResource.java @@ -46,7 +46,6 @@ public class UserResource { @EJB private UserResourceFacade userLocalEJB; - //private Logger logger = Logger.getLogger(UserResource.class.getName()); private Logger logger = Logger.getLogger(UserResource.class); public UserResource() { @@ -64,7 +63,7 @@ public Response addUser(UsersDTO user) { UsersDTO newUser = new UsersDTO(null, null, null); return Response.ok(newUser).build(); } else { - System.out.println("Created new user with id = " + createdUser.getUserID() + ", name = " + createdUser.getName()); + logger.info("Created new user with id = " + createdUser.getUserID() + ", name = " + createdUser.getName()); return Response.ok(createdUser).build(); } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CollectionResourceImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CollectionResourceImp.java index b19e628aa..1fb8c1d5d 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CollectionResourceImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CollectionResourceImp.java @@ -13,6 +13,8 @@ import javax.persistence.PersistenceContext; import javax.persistence.Query; +import org.apache.log4j.Logger; + import qa.qcri.aidr.predictui.entities.AidrCollection; import qa.qcri.aidr.predictui.facade.CollectionResourceFacade; @@ -23,6 +25,8 @@ @Stateless public class CollectionResourceImp implements CollectionResourceFacade { + private Logger logger = Logger.getLogger(CollectionResourceImp.class); + @PersistenceContext(unitName = "qa.qcri.aidr.collectorManager-PU") private EntityManager em; @@ -46,6 +50,7 @@ public List getAllRunningCollectionsByUserID(int userID) { return collections; } catch (NoResultException e) { + logger.warn("No result."); return null; } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisManagementResourceFacadeImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisManagementResourceFacadeImp.java index 2bf2c5afb..e659daf20 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisManagementResourceFacadeImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisManagementResourceFacadeImp.java @@ -44,7 +44,6 @@ public String trashByCrisisCode(String crisisCode) { // 3. remove tasks for this crisisID from document table --> // this will trigger deletion of documents for this crisisID from // the task_assignment table through DELETE CASCADE - System.out.println("[trashByCrisisCode] Received request to trash collection from aidr_predict DB: " + crisisCode); logger.info("Received request to trash collection from aidr_predict DB: " + crisisCode); try { @@ -78,7 +77,6 @@ public String trashByCrisisCode(String crisisCode) { List associatedDocumentsDTO = remoteDocumentEJB.findUnLabeledDocumentsByCrisisID(crisis.getCrisisID()); if (associatedDocumentsDTO.isEmpty()) { StringBuilder sb = new StringBuilder().append("{\"TRASHED\":").append(crisis.getCrisisID()).append("}"); - logger.info("Success in deleting crisis: " + crisisCode); return sb.toString(); } logger.info("Found for " + crisisCode + ", unlabeled docs to delete = " + associatedDocumentsDTO.size()); @@ -114,7 +112,6 @@ public String untrashByCrisisCode(String crisisCode) { //TODO: // 1. set aidr_predict.crisis.isTrashed = false // 2. set model_family.isActive = 1 - System.out.println("Received request to untrash collection from aidr_predict DB: " + crisisCode); logger.info("Received request to untrash collection from aidr_predict DB: " + crisisCode); CrisisDTO crisis = null; diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisTypeResourceImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisTypeResourceImp.java index d6878c9df..4c1c76654 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisTypeResourceImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisTypeResourceImp.java @@ -9,6 +9,8 @@ import javax.ejb.EJB; import javax.ejb.Stateless; +import org.apache.log4j.Logger; + import qa.qcri.aidr.common.exception.PropertyNotSetException; import qa.qcri.aidr.dbmanager.dto.CrisisTypeDTO; import qa.qcri.aidr.predictui.facade.CrisisTypeResourceFacade; @@ -23,6 +25,8 @@ public class CrisisTypeResourceImp implements CrisisTypeResourceFacade { //@PersistenceContext(unitName = "qa.qcri.aidr.predictui-EJBS") //private EntityManager em; + private Logger logger = Logger.getLogger(CrisisTypeResourceImp.class); + @EJB private qa.qcri.aidr.dbmanager.ejb.remote.facade.CrisisTypeResourceFacade remoteCrisisTypeEJB; @@ -31,8 +35,7 @@ public List getCrisisTypes() { List dtoList = remoteCrisisTypeEJB.getAllCrisisTypes(); return dtoList; } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getCrisisTypes."); } return null; @@ -64,51 +67,30 @@ public CrisisTypeDTO addCrisisType(CrisisTypeDTO crisisType) { try { return remoteCrisisTypeEJB.addCrisisType(crisisType); } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in addCrisisType."); } return null; - //em.persist(crisisType); - //return crisisType; } public CrisisTypeDTO getCrisisTypeByID(Long id) { try { return remoteCrisisTypeEJB.findCrisisTypeByID(id); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getCrisisTypeByID for id : " + id); } return null; - /* - Query query = em.createNamedQuery("CrisisType.findByCrisisTypeID", CrisisType.class); - query.setParameter("crisisTypeID", id); - return (CrisisType) query.getSingleResult(); - */ } public CrisisTypeDTO editCrisisType(CrisisTypeDTO crisis) { try { return remoteCrisisTypeEJB.editCrisisType(crisis); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in editCrisisType."); } return null; - //CrisisType newCrisisType = em.merge(crisis); - //return newCrisisType; } public void deleteCrisisType(Long id) { remoteCrisisTypeEJB.deleteCrisisType(id); - - /* - CrisisType crisisType = em.find(CrisisType.class, id); - if (crisisType != null) { - em.remove(crisisType); - } - else{ - throw new RuntimeException(); - }*/ } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CustomUITemplateFacadeImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CustomUITemplateFacadeImp.java index 1808d522b..dab0d518f 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CustomUITemplateFacadeImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CustomUITemplateFacadeImp.java @@ -31,6 +31,7 @@ public List getAllCustomUITemplateByCrisisID(long crisisID) List customUITemplates = remoteCustomUiTemplateEJB.getAllCustomUITemplateByCrisisID(crisisID); return customUITemplates; } catch (Exception e) { + logger.error("Error in getAllCustomUITemplateByCrisisID for crisis : " + crisisID); return null; } } @@ -43,6 +44,8 @@ public List getCustomUITemplateBasedOnTypeByCrisisID(long c List customUITemplates = remoteCustomUiTemplateEJB.getCustomUITemplateBasedOnTypeByCrisisID(crisisID, templateType); return customUITemplates; } catch (Exception e) { + logger.error("Error in getCustomUITemplateBasedOnTypeByCrisisID for crisis id : " + crisisID + + " and templateType : " + templateType); return null; } } @@ -53,6 +56,8 @@ public List getCustomUITemplateBasedOnTypeByCrisisIDAndAttr List customUITemplates = remoteCustomUiTemplateEJB.getCustomUITemplateBasedOnTypeByCrisisIDAndAttributeID(crisisID, attributeID, templateType); return customUITemplates; } catch (Exception e) { + logger.error("Error in getCustomUITemplateBasedOnTypeByCrisisIDAndAttributeID " + + "for crisisId : " + " attributeId : " + attributeID + " and templateType : " + templateType); return null; } } @@ -63,6 +68,8 @@ public List getCustomUITemplateByCrisisIDAndAttributeID(lon List customUITemplates = remoteCustomUiTemplateEJB.getCustomUITemplateByCrisisIDAndAttributeID(crisisID, attributeID); return customUITemplates; } catch (Exception e) { + logger.error("Error in getCustomUITemplateByCrisisIDAndAttributeID for crisisID : " + crisisID + + " and attributeID : " + attributeID); return null; } } @@ -73,6 +80,7 @@ public CustomUiTemplateDTO addCustomUITemplate(CustomUiTemplateDTO customUITempl CustomUiTemplateDTO dto = remoteCustomUiTemplateEJB.addCustomUITemplate(customUITemplate); return dto; } catch (Exception e) { + logger.error("Error in addCustomUITemplate."); return null; } } @@ -83,6 +91,7 @@ public CustomUiTemplateDTO updateCustomUITemplate(CustomUiTemplateDTO currentTem CustomUiTemplateDTO dto = remoteCustomUiTemplateEJB.updateCustomUITemplate(currentTemplate, updatedTemplate); return dto; } catch (Exception e) { + logger.error("Error in updateCustomUITemplate."); return null; } } @@ -93,6 +102,7 @@ public CustomUiTemplateDTO updateCustomUITemplateStatus(CustomUiTemplateDTO curr CustomUiTemplateDTO dto = remoteCustomUiTemplateEJB.updateCustomUITemplateStatus(currentTemplate, updatedTemplate); return dto; } catch (Exception e) { + logger.error("Error in updateCustomUITemplateStatus."); return null; } } @@ -102,7 +112,8 @@ public void deleteCustomUITemplateBasedOnTypeByCrisisID(long crisisID, int type) try { remoteCustomUiTemplateEJB.deleteCustomUITemplateBasedOnTypeByCrisisID(crisisID, type); } catch (Exception e) { - logger.error("exception", e); + logger.error("Error in deleteCustomUITemplateBasedOnTypeByCrisisID for crisisID " + + " : " + crisisID + " and type : " + type, e); } } @@ -111,7 +122,7 @@ public void deleteCustomUITemplateByCrisisID(long crisisID) { try { remoteCustomUiTemplateEJB.deleteCustomUITemplateByCrisisID(crisisID); } catch (Exception e) { - logger.error("exception", e); + logger.error("Error in deleteCustomUITemplateByCrisisID for crisisID : " + crisisID, e); } } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/DocumentFacadeImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/DocumentFacadeImp.java index e940af6ba..926933eca 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/DocumentFacadeImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/DocumentFacadeImp.java @@ -91,7 +91,7 @@ public ResponseWrapper removeTrainingExample(Long documentID) { "Error while deleting training example id " + documentID); } } catch (Exception e) { - logger.error("exception", e); + logger.error("Error while deleting training example id " + documentID, e); return new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_FAILED), "Error while deleting training example id " + documentID); } @@ -103,8 +103,7 @@ public Integer getUnlabeledDocumentsCountByCrisisID(Long crisisID) { try { count = remoteDocument.getUnlabeledDocumentsCountByCrisisID(crisisID); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getUnlabeledDocumentsCountByCrisisID for crisis : " + crisisID, e); } return count; } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceFacadeImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceFacadeImp.java index de021fce2..d25378d74 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceFacadeImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/MiscResourceFacadeImp.java @@ -4,18 +4,9 @@ */ package qa.qcri.aidr.predictui.facade.imp; -import qa.qcri.aidr.common.values.DownloadType; -import qa.qcri.aidr.dbmanager.dto.CrisisDTO; -import qa.qcri.aidr.dbmanager.dto.DocumentDTO; -import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentDTO; -import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentList; -import qa.qcri.aidr.dbmanager.dto.taggerapi.HumanLabeledDocumentListWrapper; -import qa.qcri.aidr.dbmanager.dto.taggerapi.ItemToLabelDTO; -import qa.qcri.aidr.dbmanager.dto.taggerapi.TrainingDataDTO; -import qa.qcri.aidr.predictui.facade.MiscResourceFacade; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; -import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; -import qa.qcri.aidr.task.ejb.TaskManagerRemote; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import javax.ejb.EJB; import javax.ejb.Stateless; @@ -29,11 +20,18 @@ import org.apache.log4j.Logger; import org.glassfish.jersey.jackson.JacksonFeature; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import qa.qcri.aidr.common.values.DownloadType; +import qa.qcri.aidr.dbmanager.dto.CrisisDTO; +import qa.qcri.aidr.dbmanager.dto.DocumentDTO; +import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentDTO; +import qa.qcri.aidr.dbmanager.dto.HumanLabeledDocumentList; +import qa.qcri.aidr.dbmanager.dto.taggerapi.HumanLabeledDocumentListWrapper; +import qa.qcri.aidr.dbmanager.dto.taggerapi.ItemToLabelDTO; +import qa.qcri.aidr.dbmanager.dto.taggerapi.TrainingDataDTO; +import qa.qcri.aidr.predictui.facade.MiscResourceFacade; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurationProperty; +import qa.qcri.aidr.predictui.util.TaggerAPIConfigurator; +import qa.qcri.aidr.task.ejb.TaskManagerRemote; /** * @@ -91,8 +89,7 @@ public List getHumanLabeledDocumentsByCrisisID(Long cri try { return remoteTaskManager.getHumanLabeledDocumentsByCrisisID(crisisID, count); } catch (Exception e) { - logger.error("exception for crisisID = " + crisisID); - e.printStackTrace(); + logger.error("exception for crisisID = " + crisisID, e); return null; } } @@ -104,8 +101,6 @@ public List getHumanLabeledDocumentsByCrisisCode(String } catch (Exception e) { logger.error("Exception for crisis code = " + crisisCode); logger.error("Exception", e); - //System.out.println("exception for crisis code = " + crisisCode); - //e.printStackTrace(); return null; } } @@ -115,8 +110,7 @@ public List getHumanLabeledDocumentsByCrisisIDUserID(Lo try { return remoteTaskManager.getHumanLabeledDocumentsByCrisisIDUserID(crisisID, userID, count); } catch (Exception e) { - logger.error("exception for crisisID = " + crisisID + ", userID = " + userID); - e.printStackTrace(); + logger.error("exception for crisisID = " + crisisID + ", userID = " + userID, e); return null; } } @@ -126,8 +120,7 @@ public List getHumanLabeledDocumentsByCrisisIDUserName( try { return remoteTaskManager.getHumanLabeledDocumentsByCrisisIDUserName(crisisID, userName, count); } catch (Exception e) { - logger.error("exception for crisisID = " + crisisID + ", userName = " + userName); - e.printStackTrace(); + logger.error("exception for crisisID = " + crisisID + ", userName = " + userName, e); return null; } } @@ -142,8 +135,7 @@ public List getHumanLabeledDocumentsByCrisisCodeUserNam return null; } } catch (Exception e) { - logger.error("exception for crisis = " + crisisCode + ", userName = " + userName); - e.printStackTrace(); + logger.error("exception for crisis = " + crisisCode + ", userName = " + userName, e); return null; } } @@ -152,9 +144,9 @@ public List getHumanLabeledDocumentsByCrisisCodeUserNam public String downloadItems(HumanLabeledDocumentList dtoList, String queryString, String crisisCode, String userName, Integer count, String fileType, String contentType) { + String errorMsg = "Exception in generating file from human labeled items"; try { HumanLabeledDocumentListWrapper postBody = new HumanLabeledDocumentListWrapper(dtoList, queryString); - //System.out.println("Request POST body: " + postBody.toJsonString()); logger.info("Received request to create file for: " + dtoList.getTotal() + "items for crisis = " + crisisCode + "userName = " + userName); Client client = ClientBuilder.newBuilder().register(JacksonFeature.class).build(); @@ -164,17 +156,16 @@ public String downloadItems(HumanLabeledDocumentList dtoList, String queryString logger.info("Invoking REST call: " + persisterMainUrl + targetAPI + "collectionCode=" + crisisCode + "&userName=" + userName); Response clientResponse = webResource.request(MediaType.APPLICATION_JSON) .post(Entity.json(postBody), Response.class); - //System.out.println("received raw response from persister: " + clientResponse); Map jsonResponse = clientResponse.readEntity(Map.class); logger.info("Received response from persister: " + jsonResponse); if (jsonResponse.get("url") != null) { return jsonResponse.get("url").toString(); } else { - return new String("Exception in generating file from human labeled items"); + return errorMsg; } } catch (Exception e) { - e.printStackTrace(); - return new String("Exception in generating file from human labeled items"); + logger.error(errorMsg + " for queryString : " + queryString); + return errorMsg; } } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/ModelFacadeImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/ModelFacadeImp.java index 47ce3e3ca..05d701639 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/ModelFacadeImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/ModelFacadeImp.java @@ -42,7 +42,7 @@ public List getAllModels() { try { return remoteModelEJB.getAllModels(); } catch (PropertyNotSetException e) { - e.printStackTrace(); + logger.error("Error in getAllModels.", e); } return null; diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/NominalLabelResourceImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/NominalLabelResourceImp.java index 3f08827fa..2820642ef 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/NominalLabelResourceImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/NominalLabelResourceImp.java @@ -9,7 +9,7 @@ import javax.ejb.EJB; import javax.ejb.Stateless; -import org.hibernate.Hibernate; +import org.apache.log4j.Logger; import qa.qcri.aidr.common.exception.PropertyNotSetException; import qa.qcri.aidr.dbmanager.dto.NominalLabelDTO; @@ -24,7 +24,8 @@ @Stateless public class NominalLabelResourceImp implements NominalLabelResourceFacade { - + private Logger logger = Logger.getLogger(NominalLabelResourceImp.class); + @EJB private qa.qcri.aidr.dbmanager.ejb.remote.facade.NominalLabelResourceFacade remoteNominalLabelEJB; @@ -36,8 +37,7 @@ public NominalLabelDTO addNominalLabel(NominalLabelDTO label) { try { return remoteNominalLabelEJB.addNominalLabel(label); } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in addNominalLabel.", e); return null; } } @@ -47,8 +47,7 @@ public NominalLabelDTO getNominalLabelByID(Long id) { try { return remoteNominalLabelEJB.getNominalLabelByID(id); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getNominalLabelByID for id : " + id, e); return null; } } @@ -58,8 +57,7 @@ public NominalLabelDTO editNominalLabel(NominalLabelDTO label) { try { return remoteNominalLabelEJB.editNominalLabel(label); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in editNominalLabel for label code : " + label.getNominalLabelCode(), e ); return null; } } @@ -69,8 +67,7 @@ public List getAllNominalLabel() { try { return remoteNominalLabelEJB.getAllNominalLabels(); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in getAllNominalLabel.", e); return null; } } @@ -80,8 +77,7 @@ public void deleteNominalLabel(Long labelID) { try { remoteNominalLabelEJB.deleteNominalLabelByID(labelID); } catch (PropertyNotSetException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + logger.error("Error in deleteNominalLabel by ID : " + labelID, e); } } @@ -96,10 +92,7 @@ public void deleteNominalLabelDataByAttribute(Long attributeID) { deleteNominalLabel(nominalLabelDTO.getNominalLabelId()); } }catch (Exception e) { - e.printStackTrace(); + logger.error("Error in deleteNominalLabelDataByAttribute for attributeID : " + attributeID, e); } - } - - } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/SystemEventFacadeImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/SystemEventFacadeImp.java index 469d9ade6..db66636ab 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/SystemEventFacadeImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/SystemEventFacadeImp.java @@ -3,12 +3,16 @@ import javax.ejb.EJB; import javax.ejb.Stateless; +import org.apache.log4j.Logger; + import qa.qcri.aidr.dbmanager.dto.SystemEventDTO; import qa.qcri.aidr.predictui.facade.SystemEventFacade; @Stateless public class SystemEventFacadeImp implements SystemEventFacade{ + private Logger logger = Logger.getLogger(SystemEventFacadeImp.class); + @EJB private qa.qcri.aidr.dbmanager.ejb.remote.facade.SystemEventResourceFacade remotesystemEJB; @@ -33,7 +37,7 @@ public void insertSystemEvent(String severity, String module, String code, @Override public void insertSystemEvent(String severity, String module, String code, String description, Boolean emailSent) { - System.out.println("inserting into sys event"); + logger.info("inserting into sys event"); remotesystemEJB.insertSystemEvent(severity, module, code, description, emailSent); } diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/UserResourceImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/UserResourceImp.java index 598fa8639..2e10c89ea 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/UserResourceImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/UserResourceImp.java @@ -71,7 +71,6 @@ public List getAllUsers() { List dbUsers = new ArrayList(); try { dbUsers = remoteUsersEJB.getAllUsers(); - System.out.println("Fetched users list: " + dbUsers.size()); return dbUsers; } catch (Exception e) { logger.error("exception:", e); From cd4493e0ba896169c0568086ee2487a7684da33a Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Mon, 3 Aug 2015 13:36:11 +0530 Subject: [PATCH 35/44] Minor code fix. --- .../java/qa/qcri/aidr/predictui/api/TestDBManagerResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TestDBManagerResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TestDBManagerResource.java index afacaa160..e1de4877e 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TestDBManagerResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/TestDBManagerResource.java @@ -358,7 +358,7 @@ public Response TestAddDeleteDocument() { if (ret != null && ret.intValue() == 1) return Response.ok("Document Add-Delete test successful" + dto).build(); } } catch (Exception e) { - logger.error("Error in /document/adddel." e); + logger.error("Error in /document/adddel.",e); return Response.ok("Exception: " + e).build(); } return Response.ok("Document add-Delete test failed").build(); From d1687cd0a4d650b86301109cbdcb76c979e15168 Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Mon, 3 Aug 2015 15:33:29 +0530 Subject: [PATCH 36/44] Commit for fixing deployment issues in aidr-analytics and db-manager --- .../analysis/api/WriteStatisticsData.java | 6 ++-- .../utils/AnalyticsConfigurationProperty.java | 4 ++- .../analysis/utils/AnalyticsConfigurator.java | 2 +- .../analysis/utils/ChannelBufferManager.java | 12 +++---- .../analysis/utils/JedisConnectionObject.java | 6 ++-- .../utils/OutputConfigurationProperty.java | 22 ------------ .../analysis/utils/OutputConfigurator.java | 35 ------------------- .../src/main/resources/config.properties | 2 +- .../dbmanager/entities/misc/SystemEvent.java | 4 --- 9 files changed, 16 insertions(+), 77 deletions(-) delete mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java delete mode 100644 aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurator.java diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java index 4dc6391b4..0d2490a48 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/api/WriteStatisticsData.java @@ -1,7 +1,6 @@ package qa.qcri.aidr.analysis.api; -import java.util.Date; import java.util.List; import java.util.Map; import java.util.Set; @@ -25,15 +24,14 @@ import qa.qcri.aidr.analysis.entity.ConfidenceData; import qa.qcri.aidr.analysis.entity.TagData; +import qa.qcri.aidr.analysis.facade.ConfidenceStatisticsResourceFacade; +import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; import qa.qcri.aidr.analysis.stat.ConfDataMapRecord; import qa.qcri.aidr.analysis.stat.TagDataMapRecord; import qa.qcri.aidr.analysis.utils.AnalyticsConfigurator; import qa.qcri.aidr.analysis.utils.ChannelBufferManager; import qa.qcri.aidr.analysis.utils.ConfCounterKey; import qa.qcri.aidr.analysis.utils.CounterKey; -import qa.qcri.aidr.analysis.utils.OutputConfigurator; -import qa.qcri.aidr.analysis.facade.ConfidenceStatisticsResourceFacade; -import qa.qcri.aidr.analysis.facade.TagDataStatisticsResourceFacade; import qa.qcri.aidr.common.values.ReturnCode; diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurationProperty.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurationProperty.java index ddaf68138..7d1769ba2 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurationProperty.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurationProperty.java @@ -14,7 +14,9 @@ */ public enum AnalyticsConfigurationProperty implements ConfigurationProperty { - GRANULARITY("granularity"), TAGGER_REST_URI("TAGGER_REST_URI"); + GRANULARITY("granularity"), TAGGER_REST_URI("TAGGER_REST_URI"), REDIS_HOST("REDIS_HOST"), REDIS_PORT("REDIS_PORT"), LOGGER("logger"), + PERSISTER_LOAD_LIMIT("PERSISTER_LOAD_LIMIT"), PERSISTER_LOAD_CHECK_INTERVAL_MINUTES("PERSISTER_LOAD_CHECK_INTERVAL_MINUTES"), + MAX_MESSAGES_COUNT("MAX_MESSAGES_COUNT"), TAGGER_CHANNEL_BASENAME("TAGGER_CHANNEL_BASENAME"); private final String configurationProperty; diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurator.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurator.java index 1a9e207e6..169c8cc04 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurator.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/AnalyticsConfigurator.java @@ -17,7 +17,7 @@ public class AnalyticsConfigurator extends BaseConfigurator { private static final Logger logger = Logger .getLogger(AnalyticsConfigurator.class); - public static String configLoadFileName = "config.properties"; + public static final String configLoadFileName = "config.properties"; private static final AnalyticsConfigurator instance = new AnalyticsConfigurator(); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java index ea06c6415..4b155d5af 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/ChannelBufferManager.java @@ -89,13 +89,13 @@ public class ChannelBufferManager { public ChannelBufferManager() {} public void initiateChannelBufferManager(final String channelRegEx) { - Configurator configurator = OutputConfigurator.getInstance(); + Configurator configurator = AnalyticsConfigurator.getInstance(); redisLoadShedder = new ConcurrentHashMap(); - redisHost = configurator.getProperty(OutputConfigurationProperty.REDIS_HOST); - redisPort = Integer.parseInt(configurator.getProperty(OutputConfigurationProperty.REDIS_PORT)); - CHANNEL_PREFIX_STRING = configurator.getProperty(OutputConfigurationProperty.TAGGER_CHANNEL_BASENAME)+"."; - PERSISTER_LOAD_CHECK_INTERVAL_MINUTES = Integer.parseInt(configurator.getProperty(OutputConfigurationProperty.PERSISTER_LOAD_CHECK_INTERVAL_MINUTES)); - PERSISTER_LOAD_LIMIT = Integer.parseInt(configurator.getProperty(OutputConfigurationProperty.PERSISTER_LOAD_LIMIT)); + redisHost = configurator.getProperty(AnalyticsConfigurationProperty.REDIS_HOST); + redisPort = Integer.parseInt(configurator.getProperty(AnalyticsConfigurationProperty.REDIS_PORT)); + CHANNEL_PREFIX_STRING = configurator.getProperty(AnalyticsConfigurationProperty.TAGGER_CHANNEL_BASENAME)+"."; + PERSISTER_LOAD_CHECK_INTERVAL_MINUTES = Integer.parseInt(configurator.getProperty(AnalyticsConfigurationProperty.PERSISTER_LOAD_CHECK_INTERVAL_MINUTES)); + PERSISTER_LOAD_LIMIT = Integer.parseInt(configurator.getProperty(AnalyticsConfigurationProperty.PERSISTER_LOAD_LIMIT)); AnalyticsConfigurator analyticsConfigurator = AnalyticsConfigurator.getInstance(); granularityList = analyticsConfigurator .getGranularities(); diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java index 76a0f2249..772f1cacd 100644 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java +++ b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/JedisConnectionObject.java @@ -23,9 +23,9 @@ public class JedisConnectionObject { private static JedisPool pool = null; // only one JedisPool per servlet instance private Jedis subscriberJedis = null; // one for each GET request - private static OutputConfigurator configProperties = OutputConfigurator.getInstance(); - public static String redisHost = configProperties.getProperty(OutputConfigurationProperty.REDIS_HOST); - public static int redisPort = Integer.valueOf(configProperties.getProperty(OutputConfigurationProperty.REDIS_PORT)); + private static AnalyticsConfigurator configProperties = AnalyticsConfigurator.getInstance(); + public static String redisHost = configProperties.getProperty(AnalyticsConfigurationProperty.REDIS_HOST); + public static int redisPort = Integer.valueOf(configProperties.getProperty(AnalyticsConfigurationProperty.REDIS_PORT)); private static boolean poolSetup = false; private boolean connectionSetup = false; diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java deleted file mode 100644 index d3dfc90d6..000000000 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurationProperty.java +++ /dev/null @@ -1,22 +0,0 @@ -package qa.qcri.aidr.analysis.utils; - -import qa.qcri.aidr.common.code.ConfigurationProperty; - -public enum OutputConfigurationProperty implements ConfigurationProperty { - - REDIS_HOST("REDIS_HOST"), REDIS_PORT("REDIS_PORT"), LOGGER("logger"), - PERSISTER_LOAD_LIMIT("PERSISTER_LOAD_LIMIT"), PERSISTER_LOAD_CHECK_INTERVAL_MINUTES("PERSISTER_LOAD_CHECK_INTERVAL_MINUTES"), - MAX_MESSAGES_COUNT("MAX_MESSAGES_COUNT"), TAGGER_CHANNEL_BASENAME("TAGGER_CHANNEL_BASENAME"); - - private final String configurationProperty; - - private OutputConfigurationProperty(String property) { - configurationProperty = property; - } - - @Override - public String getName() { - return this.configurationProperty; - } - -} diff --git a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurator.java b/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurator.java deleted file mode 100644 index 2e1492845..000000000 --- a/aidr-analytics/src/main/java/qa/qcri/aidr/analysis/utils/OutputConfigurator.java +++ /dev/null @@ -1,35 +0,0 @@ -package qa.qcri.aidr.analysis.utils; - -import org.apache.log4j.Logger; - -import qa.qcri.aidr.common.code.impl.BaseConfigurator; -import qa.qcri.aidr.common.exception.ConfigurationPropertyFileException; -import qa.qcri.aidr.common.exception.ConfigurationPropertyNotRecognizedException; -import qa.qcri.aidr.common.exception.ConfigurationPropertyNotSetException; - -/** - * User: yakubenkova.elena@gmail.com Date: 29.09.14 - */ - -public class OutputConfigurator extends BaseConfigurator { - - private static final Logger LOGGER = Logger - .getLogger(OutputConfigurator.class); - - public static final String configLoadFileName = "config.properties"; - - private static final OutputConfigurator instance = new OutputConfigurator(); - - private OutputConfigurator() { - LOGGER.info("Initializing OutputConfigurator."); - this.initProperties(configLoadFileName, OutputConfigurationProperty.values()); - } - - public static OutputConfigurator getInstance() - throws ConfigurationPropertyNotSetException, - ConfigurationPropertyNotRecognizedException, - ConfigurationPropertyFileException { - return instance; - } - -} diff --git a/aidr-analytics/src/main/resources/config.properties b/aidr-analytics/src/main/resources/config.properties index 777af4d5a..26d5c8d15 100644 --- a/aidr-analytics/src/main/resources/config.properties +++ b/aidr-analytics/src/main/resources/config.properties @@ -1,5 +1,5 @@ # -# Configuration for aidr-output +# Configuration for aidr-analytics # ############################# diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/entities/misc/SystemEvent.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/entities/misc/SystemEvent.java index 42f7b0697..e8286f160 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/entities/misc/SystemEvent.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/entities/misc/SystemEvent.java @@ -12,8 +12,6 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; -import org.apache.log4j.Logger; - @Entity @Table(name = "system_event", catalog = "aidr_predict") public class SystemEvent implements java.io.Serializable{ @@ -22,7 +20,6 @@ public class SystemEvent implements java.io.Serializable{ * */ private static final long serialVersionUID = 6475156575527664114L; - private Logger logger = Logger.getLogger(SystemEvent.class); public SystemEvent(String severity, String module, String description) { this(severity,module,null,description); @@ -54,7 +51,6 @@ public SystemEvent(Date receivedAt, String severity, String module, String code, public SystemEvent(Date receivedAt, String severity, String module, String code, String description, Boolean emailSent) { super(); - logger.info("date:" + receivedAt + " modeule:" + module +" emailSent?" + emailSent); this.receivedAt = receivedAt; this.severity = severity; this.module = module; From 2e2222ff36e015d622a27fbbedc9ab0f8a4a60c3 Mon Sep 17 00:00:00 2001 From: Latika Bhurani Date: Mon, 3 Aug 2015 18:00:51 +0530 Subject: [PATCH 37/44] Commit for logging : changing logger instance to static final in dtos --- .../src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisTypeDTO.java | 2 +- .../src/main/java/qa/qcri/aidr/dbmanager/dto/DocumentDTO.java | 2 +- .../qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java | 2 +- .../qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java | 2 +- .../src/main/java/qa/qcri/aidr/dbmanager/dto/ModelDTO.java | 2 +- .../java/qa/qcri/aidr/dbmanager/dto/ModelNominalLabelIdDTO.java | 2 +- .../aidr/dbmanager/dto/NominalAttributeDependentLabelIdDTO.java | 2 +- .../qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataDTO.java | 2 +- .../qcri/aidr/dbmanager/dto/NominalLabelTrainingDataIdDTO.java | 2 +- .../src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAnswerDTO.java | 2 +- .../main/java/qa/qcri/aidr/dbmanager/dto/TaskAssignmentDTO.java | 2 +- .../ejb/remote/facade/imp/CrisisResourceFacadeImp.java | 2 +- .../ejb/remote/facade/imp/CrisisTypeResourceFacadeImp.java | 2 +- .../qa/qcri/aidr/predictui/facade/imp/CrisisResourceImp.java | 2 +- .../qcri/aidr/predictui/facade/imp/CrisisTypeResourceImp.java | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisTypeDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisTypeDTO.java index ff2ef7541..ffbe77fb0 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisTypeDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/CrisisTypeDTO.java @@ -19,7 +19,7 @@ public class CrisisTypeDTO implements java.io.Serializable { private static final long serialVersionUID = 8074463052776843105L; - private Logger logger = Logger.getLogger(CrisisTypeDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private Long crisisTypeId; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/DocumentDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/DocumentDTO.java index 895d7f509..5877253c3 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/DocumentDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/DocumentDTO.java @@ -32,7 +32,7 @@ public class DocumentDTO implements Serializable { * */ private static final long serialVersionUID = 6833511192868020105L; - private Logger logger = Logger.getLogger(DocumentDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private Long documentID; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java index bc86310a0..988d3be66 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentDTO.java @@ -28,7 +28,7 @@ public class HumanLabeledDocumentDTO implements Serializable { * */ private static final long serialVersionUID = -8284975892012345464L; - private Logger logger = Logger.getLogger(HumanLabeledDocumentDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private DocumentDTO doc; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java index d25105ab7..b909bb015 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/HumanLabeledDocumentList.java @@ -21,7 +21,7 @@ public class HumanLabeledDocumentList implements Serializable { * */ private static final long serialVersionUID = 4864042221879747333L; - private Logger logger = Logger.getLogger(HumanLabeledDocumentList.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private List items; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelDTO.java index a2622b62c..2ffcc446f 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelDTO.java @@ -27,7 +27,7 @@ public class ModelDTO implements Serializable { * */ private static final long serialVersionUID = 436159208494993271L; - private Logger logger = Logger.getLogger(ModelDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private Long modelId; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelNominalLabelIdDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelNominalLabelIdDTO.java index a510fb107..181cb9217 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelNominalLabelIdDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/ModelNominalLabelIdDTO.java @@ -20,7 +20,7 @@ public class ModelNominalLabelIdDTO implements Serializable { * */ private static final long serialVersionUID = -7966215874963999322L; - private Logger logger = Logger.getLogger(ModelNominalLabelIdDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private Long modelId; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalAttributeDependentLabelIdDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalAttributeDependentLabelIdDTO.java index 14e1f3941..3b275a92f 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalAttributeDependentLabelIdDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalAttributeDependentLabelIdDTO.java @@ -19,7 +19,7 @@ public class NominalAttributeDependentLabelIdDTO implements Serializable { * */ private static final long serialVersionUID = 5910754106878087122L; - private Logger logger = Logger.getLogger(NominalAttributeDependentLabelIdDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private Long nominalAttributeId; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataDTO.java index 4a63d8274..4854b1bf4 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataDTO.java @@ -20,7 +20,7 @@ public class NominalLabelTrainingDataDTO implements Serializable { * */ private static final long serialVersionUID = -6001563823600724945L; - private Logger logger = Logger.getLogger(NominalLabelTrainingDataDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private NominalLabelTrainingDataIdDTO idDTO; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataIdDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataIdDTO.java index 654d6f6ab..305c7e955 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataIdDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/NominalLabelTrainingDataIdDTO.java @@ -20,7 +20,7 @@ public class NominalLabelTrainingDataIdDTO implements Serializable { * */ private static final long serialVersionUID = 4397588498587349888L; - private Logger logger = Logger.getLogger(NominalLabelTrainingDataIdDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private Long documentId; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAnswerDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAnswerDTO.java index f9e65dc6d..323676ba9 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAnswerDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAnswerDTO.java @@ -23,7 +23,7 @@ public class TaskAnswerDTO implements Serializable { * */ private static final long serialVersionUID = 5525860415454423643L; - private Logger logger = Logger.getLogger(TaskAnswerDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private Long taskID; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAssignmentDTO.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAssignmentDTO.java index 5dcc84d21..425354897 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAssignmentDTO.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/TaskAssignmentDTO.java @@ -21,7 +21,7 @@ public class TaskAssignmentDTO implements Serializable { * */ private static final long serialVersionUID = 6766433678441426060L; - private Logger logger = Logger.getLogger(TaskAssignmentDTO.class); + private static final Logger logger = Logger.getLogger("db-manager-log"); @XmlElement private DocumentDTO document; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java index 4bc589e55..d73aa4698 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisResourceFacadeImp.java @@ -30,7 +30,7 @@ @Stateless(name="CrisisResourceFacadeImp") public class CrisisResourceFacadeImp extends CoreDBServiceFacadeImp implements CrisisResourceFacade { - private static Logger logger = Logger.getLogger("db-manager-log"); + private static final Logger logger = Logger.getLogger("db-manager-log"); @EJB private UsersResourceFacade userLocalEJB; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisTypeResourceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisTypeResourceFacadeImp.java index 3d69c81d3..e99dc03ba 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisTypeResourceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/remote/facade/imp/CrisisTypeResourceFacadeImp.java @@ -28,7 +28,7 @@ @Stateless(name = "CrisisTypeResourceFacadeImp") public class CrisisTypeResourceFacadeImp extends CoreDBServiceFacadeImp implements CrisisTypeResourceFacade { - private Logger logger = Logger.getLogger("db-manager-log"); + private static final Logger logger = Logger.getLogger("db-manager-log"); public CrisisTypeResourceFacadeImp() { super(CrisisType.class); diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisResourceImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisResourceImp.java index e98d123ff..4ce86f04f 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisResourceImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisResourceImp.java @@ -23,7 +23,7 @@ @Stateless public class CrisisResourceImp implements CrisisResourceFacade { //private static Logger logger = Logger.getLogger(CrisisResourceImp.class); - private static Logger logger = Logger.getLogger(CrisisResourceImp.class); + private Logger logger = Logger.getLogger(CrisisResourceImp.class); //@PersistenceContext(unitName = "qa.qcri.aidr.predictui-EJBS") //private EntityManager em; diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisTypeResourceImp.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisTypeResourceImp.java index 4c1c76654..a68a2416c 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisTypeResourceImp.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/facade/imp/CrisisTypeResourceImp.java @@ -25,7 +25,7 @@ public class CrisisTypeResourceImp implements CrisisTypeResourceFacade { //@PersistenceContext(unitName = "qa.qcri.aidr.predictui-EJBS") //private EntityManager em; - private Logger logger = Logger.getLogger(CrisisTypeResourceImp.class); + private static Logger logger = Logger.getLogger(CrisisTypeResourceImp.class); @EJB private qa.qcri.aidr.dbmanager.ejb.remote.facade.CrisisTypeResourceFacade remoteCrisisTypeEJB; From fd30985c6f28417f76d64c23278dbf0fdaca5bbe Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Mon, 3 Aug 2015 18:53:14 +0530 Subject: [PATCH 38/44] Minor changes. Changed logger to static final --- .../dto/taggerapi/HumanLabeledDocumentListWrapper.java | 2 +- .../dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java | 2 +- .../src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/HumanLabeledDocumentListWrapper.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/HumanLabeledDocumentListWrapper.java index ccdf00d38..98def3bd1 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/HumanLabeledDocumentListWrapper.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/dto/taggerapi/HumanLabeledDocumentListWrapper.java @@ -21,7 +21,7 @@ public class HumanLabeledDocumentListWrapper implements Serializable { * */ private static final long serialVersionUID = -7413555749237029785L; - private Logger logger = Logger.getLogger(HumanLabeledDocumentListWrapper.class); + private static final Logger logger = Logger.getLogger(HumanLabeledDocumentListWrapper.class); @XmlElement HumanLabeledDocumentList dtoList; diff --git a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java index 3ad371790..a36da0522 100644 --- a/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java +++ b/aidr-db-manager/src/main/java/qa/qcri/aidr/dbmanager/ejb/local/facade/impl/CoreDBServiceFacadeImp.java @@ -36,7 +36,7 @@ public class CoreDBServiceFacadeImp entityClass; diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java index 92807e2f2..dc03dcc31 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java @@ -238,7 +238,7 @@ public Response downloadHumanLabeledDocumentsByCrisisIDUserName(String queryStri count=null; } - if (dtoList != null) { + if (dtoList != null && dtoList.size()!=0) { ResponseWrapper response = new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)); response.setTotal(dtoList.size()); HumanLabeledDocumentList list = new HumanLabeledDocumentList(dtoList); From a26e5d9403d87e73d00e703626a1bd8451a38f9e Mon Sep 17 00:00:00 2001 From: root Date: Tue, 4 Aug 2015 08:42:14 +0300 Subject: [PATCH 39/44] tagger: retrain model y/n moved back to sys.out --- .../aidr/predict/classification/nominal/ModelController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelController.java b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelController.java index ba067dd1d..e345ceeb1 100644 --- a/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelController.java +++ b/aidr-tagger/src/main/java/qa/qcri/aidr/predict/classification/nominal/ModelController.java @@ -219,7 +219,7 @@ private boolean loadModel(int eventID, int attributeID, int modelID) { String path = getModelPath(eventID, attributeID, modelID); o = weka.core.SerializationHelper.readAll(path); } catch (Exception e) { - logger.error("Could not load model from disk (crisis " + eventID + System.out.println("Could not load model from disk (crisis " + eventID + ", attribute " + attributeID + ", model " + modelID + "). Delete model reference in DB and retrain? [y/n]"); try { From 51dfc781e583e4a981ea77e02461469321e72d84 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Tue, 4 Aug 2015 12:13:06 +0530 Subject: [PATCH 40/44] Added some logs in manager and fixed a bug in JsonDeserializer --- .../controller/CollectionController.java | 2 +- .../manager/controller/ScreenController.java | 12 ++++----- .../manager/controller/TaggerController.java | 2 +- .../qa/qcri/aidr/utils/FileCompressor.java | 2 +- .../qa/qcri/aidr/utils/JsonDeserializer.java | 27 ++++++++++++------- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/CollectionController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/CollectionController.java index 1e0d10ace..aa15d4389 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/CollectionController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/CollectionController.java @@ -523,7 +523,7 @@ public Map updateAndGetRunningCollectionByUser() throws Exception try { return getUIWrapper(collectionService.updateAndGetRunningCollectionStatusByUser(userEntity.getId()),true); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while updating and getting running collection by user"); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } } diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/ScreenController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/ScreenController.java index b708fc7cc..ae1ee6e4b 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/ScreenController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/ScreenController.java @@ -191,7 +191,7 @@ public ModelAndView attributeDetails(@PathVariable(value="id") Integer id) throw taggerUserId = taggerService.isUserExistsByUsername(userName); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while getting attribute details",e); } model.addObject("id", id); model.addObject("userId", taggerUserId); @@ -239,8 +239,8 @@ public ModelAndView modelDetails(@PathVariable(value="code") String code, @PathV } catch (Exception e) { - System.out.println("e : " + e); - e.printStackTrace(); + // System.out.println("e : " + e); + logger.error("Exception while checking whether user exist by username",e); } AidrCollection collection = collectionService.findByCode(code); @@ -407,7 +407,7 @@ public ModelAndView adminHealth(Map model) throws Exception { public ModelAndView interactiveViewDownload(@PathVariable(value="code") String code) throws Exception { String userName =""; - System.out.println("interactiveViewDownload : "); + // System.out.println("interactiveViewDownload : "); if (isHasPermissionForCollection(code)){ userName = getAuthenticatedUserName(); @@ -436,7 +436,7 @@ private ModelAndView getInteractiveViewDownload(String code, String userName){ crisis = taggerService.getCrisesByCode(code); collection = collectionService.findByCode(code); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while getting interactive view download", e); } Integer crisisId = 0; @@ -456,7 +456,7 @@ private ModelAndView getInteractiveViewDownload(String code, String userName){ try { collectionCount = collectionLogService.countTotalDownloadedItemsForCollection(collectionId); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while counting total download items for collectionID: "+collectionId, e); } } if (collection.getCount() != null && (collection.getStatus() != null || RUNNING == collection.getStatus() || RUNNING_WARNING == collection.getStatus())) { diff --git a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java index c42713d55..b569a04a0 100644 --- a/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java +++ b/aidr-manager/src/main/java/qa/qcri/aidr/manager/controller/TaggerController.java @@ -555,7 +555,7 @@ public Map loadLatestTweets(@RequestParam String code, @RequestPa try { result = taggerService.loadLatestTweets(code, constraints); } catch (Exception e) { - e.printStackTrace(); + logger.error("Exception while loading latest tweets for collection: "+code,e); return getUIWrapper(false, "System is down or under maintenance. For further inquiries please contact admin."); } return getUIWrapper(result,true); diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java index c72fe8279..e15dd0d53 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java @@ -74,7 +74,7 @@ public String zip() { in.close(); zos.closeEntry(); zos.close(); - System.out.println("Done zipping file: " + this.inputFileName + ", created file = " + this.outputFileName); + //System.out.println("Done zipping file: " + this.inputFileName + ", created file = " + this.outputFileName); return this.getOutFile(); } catch (Exception e) { diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java index ed1ec16e4..669e6b425 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java @@ -762,7 +762,7 @@ public int compare(File f1, File f2) { String currentFileName = f.getName(); if (currentFileName.endsWith(".json")) { String line; - System.out.println("Reading file : " + f.getAbsolutePath()); + //System.out.println("Reading file : " + f.getAbsolutePath()); InputStream is = new FileInputStream(f.getAbsolutePath()); br = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); while ((line = br.readLine()) != null) { @@ -828,7 +828,7 @@ public int compare(File f1, File f2) { String currentFileName = f.getName(); if (currentFileName.endsWith(".json")) { String line; - System.out.println("Reading file : " + f.getAbsolutePath()); + //System.out.println("Reading file : " + f.getAbsolutePath()); InputStream is = new FileInputStream(f.getAbsolutePath()); br = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); while ((line = br.readLine()) != null) { @@ -1365,7 +1365,8 @@ public Map generateClassifiedJson2TweetIdsJSON(String collection BufferedReader br = null; String fileToDelete = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/" + "Classified_" + collectionCode + "_tweetIds" + extension; - System.out.println("Deleteing file : " + fileToDelete); + //System.out.println("Deleteing file : " + fileToDelete); + logger.info("Deleteing file : " + fileToDelete); FileSystemOperations.deleteFile(fileToDelete); // delete if there exist a csv file with same name //System.out.println(fileNames); @@ -1613,7 +1614,8 @@ public Map generateClassifiedJson2TweetIdsJSONFiltered( ReversedLinesFileReader br = null; String fileToDelete = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/" + fileName; - System.out.println("Deleteing file : " + fileToDelete + ".zip"); + //System.out.println("Deleteing file : " + fileToDelete + ".zip"); + logger.info("Deleteing file : " + fileToDelete + ".zip"); FileSystemOperations.deleteFile(fileToDelete + ".zip"); // delete if there exist a csv file with same name //System.out.println(fileNames); @@ -1692,10 +1694,11 @@ public Map generateClassifiedJson2TweetIdsJSONFiltered( } } FileCompressor compressor = new FileCompressor(folderLocation, folderLocation, fileName); + String fileToDelete = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/" + fileName; fileName = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_DOWNLOAD_URL) + collectionCode + "/" + compressor.zip(); - String fileToDelete = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/" + fileName; - System.out.println("Deleteing file : " + fileToDelete); + //System.out.println("Deleteing file : " + fileToDelete); + logger.info("Deleteing file : " + fileToDelete); FileSystemOperations.deleteFile(fileToDelete); // delete if there exist a csv file with same name return ResultStatus.getUIWrapper("fileName", fileName, "count", totalCount); @@ -1928,7 +1931,8 @@ public Map generateClassifiedList2TweetIdsCSVFiltered(String col fileName = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_DOWNLOAD_URL) + collectionCode + "/" + compressor.zip(); FileSystemOperations.deleteFile(folderLocation + "/" + fileNameforCSVGen); - System.out.println("Deleted raw file post compression: " + fileNameforCSVGen); + //System.out.println("Deleted raw file post compression: " + fileNameforCSVGen); + logger.info("Deleted raw file post compression: " + fileNameforCSVGen); return ResultStatus.getUIWrapper("fileName", fileName, "count", totalCount); } @@ -2027,7 +2031,8 @@ public String generateClassifiedList2JSON_100K_BasedOnTweetCountFiltered(String fileName = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_DOWNLOAD_URL) + collectionCode + "/" + compressor.zip(); FileSystemOperations.deleteFile(folderLocation + "/" + fileNameforGen); - System.out.println("Deleted raw file post compression: " + fileNameforGen); + //System.out.println("Deleted raw file post compression: " + fileNameforGen); + logger.info("Deleted raw file post compression: " + fileNameforGen); return fileName; } @@ -2058,7 +2063,8 @@ public Map generateClassifiedList2TweetIdsJSONFiltered(String co String fileToDelete = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.DEFAULT_PERSISTER_FILE_PATH) + collectionCode + "/" + fileNameforGen; int totalCount = 0; try { - System.out.println("Deleteing file : " + fileToDelete + ".zip"); + //System.out.println("Deleteing file : " + fileToDelete + ".zip"); + logger.info("Deleteing file : " + fileToDelete + ".zip"); FileSystemOperations.deleteFile(fileToDelete + ".zip"); // delete if there exist a file with same name StringBuffer outputFile = new StringBuffer().append(folderLocation).append("/").append(fileNameforGen); @@ -2117,7 +2123,8 @@ public Map generateClassifiedList2TweetIdsJSONFiltered(String co fileName = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_DOWNLOAD_URL) + collectionCode + "/" + compressor.zip(); FileSystemOperations.deleteFile(fileToDelete); // delete if there exist a file with same name - System.out.println("Deleted raw file post compression: " + fileToDelete); + //System.out.println("Deleted raw file post compression: " + fileToDelete); + logger.info("Deleted raw file post compression: " + fileToDelete); return ResultStatus.getUIWrapper("fileName", fileName, "count", totalCount); } From f1f5effa063be9836feea3e3fb511d86bd041ce3 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Tue, 4 Aug 2015 12:29:04 +0530 Subject: [PATCH 41/44] Fixed download link for filter/genCsv --- .../src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java index 669e6b425..4e2a56d2d 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java @@ -1831,7 +1831,7 @@ public String generateClassifiedList2CSV_100K_BasedOnTweetCountFiltered(String c // Compress generated file and send the compressed file link FileCompressor compressor = new FileCompressor(folderLocation, folderLocation, fileNameforCSVGen); fileName = PersisterConfigurator.getInstance().getProperty(PersisterConfigurationProperty.PERSISTER_DOWNLOAD_URL) - + "/" + collectionCode + "/" + compressor.zip(); + + collectionCode + "/" + compressor.zip(); FileSystemOperations.deleteFile(folderLocation + "/" + fileNameforCSVGen); logger.info("Deleted raw file post compression: " + fileNameforCSVGen); return fileName; From 83e228c6cdf2f8f50e8a376e259964fe255143b1 Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Tue, 4 Aug 2015 15:24:18 +0530 Subject: [PATCH 42/44] Added a check on ui to report 0 human tagged tweets as info not error. --- .../controller/TrainingDataController.js | 9 ++++++++- .../java/qa/qcri/aidr/utils/FileCompressor.java | 13 +++++-------- .../qa/qcri/aidr/predictui/api/MiscResource.java | 11 ++++++++--- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js b/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js index 561b30936..48516d399 100644 --- a/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js +++ b/aidr-manager/src/main/webapp/resources/js/taggui/training-data/controller/TrainingDataController.js @@ -221,7 +221,14 @@ Ext.define('TAGGUI.training-data.controller.TrainingDataController', { var resp = Ext.decode(response.responseText); if (resp.success) { if (resp.data && resp.data != '') { - me.mainComponent.downloadLink.setText('
' + resp.data.fileName + '
', false); + if(resp.data.total){ + me.mainComponent.downloadLink.setText('
' + resp.data.fileName + '
', false); + } + else{ + me.mainComponent.downloadLink.setText('', false); + AIDRFMFunctions.setAlert("Info", "No human tagged "+ COLLECTION_TYPES[TYPE]["plural"] + " available to download"); + } + } else { me.mainComponent.downloadLink.setText('', false); AIDRFMFunctions.setAlert("Error", "Generate Tweet Ids service returned empty url. For further inquiries please contact admin."); diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java index e15dd0d53..2f8c8b8e8 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/FileCompressor.java @@ -60,13 +60,12 @@ public String zip() { byte[] buffer = new byte[BUFFER_SIZE]; - try { - OutputStream fos = new BufferedOutputStream(new FileOutputStream(this.getOutputFileName()), BUFFER_SIZE); - ZipOutputStream zos = new ZipOutputStream(fos); + try(OutputStream fos = new BufferedOutputStream(new FileOutputStream(this.getOutputFileName()), BUFFER_SIZE); + ZipOutputStream zos = new ZipOutputStream(fos); + InputStream in = new BufferedInputStream(new FileInputStream(this.getInputFileName()), BUFFER_SIZE);) { + ZipEntry ze = new ZipEntry(this.getFileName()); zos.putNextEntry(ze); - - InputStream in = new BufferedInputStream(new FileInputStream(this.getInputFileName()), BUFFER_SIZE); int len; while ((len = in.read(buffer)) > 0) { zos.write(buffer, 0, len); @@ -89,10 +88,8 @@ public String zip() { */ public String unzip() { byte[] buffer = new byte[BUFFER_SIZE]; - try { + try(ZipInputStream zis = new ZipInputStream(new FileInputStream(this.getInputFileName()));) { // get the zip file content - ZipInputStream zis = new ZipInputStream(new FileInputStream(this.getInputFileName())); - // get the zipped file list entry ZipEntry ze = zis.getNextEntry(); if (ze != null) { diff --git a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java index dc03dcc31..e616cd945 100644 --- a/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java +++ b/aidr-tagger-api/src/main/java/qa/qcri/aidr/predictui/api/MiscResource.java @@ -238,12 +238,17 @@ public Response downloadHumanLabeledDocumentsByCrisisIDUserName(String queryStri count=null; } - if (dtoList != null && dtoList.size()!=0) { + if (dtoList != null) { ResponseWrapper response = new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS)); response.setTotal(dtoList.size()); HumanLabeledDocumentList list = new HumanLabeledDocumentList(dtoList); - response.setMessage(miscEJB.downloadItems(list, queryString, dtoList.get(0).getDoc().getCrisisDTO().getCode(), - userName, count, fileType, contentType)); + if(dtoList.size()!=0){ + response.setMessage(miscEJB.downloadItems(list, queryString, dtoList.get(0).getDoc().getCrisisDTO().getCode(), + userName, count, fileType, contentType)); + } + else{ + response.setMessage("Found 0 human labeled documents"); + } return Response.ok(response).build(); } else { return Response.ok(new ResponseWrapper(TaggerAPIConfigurator.getInstance().getProperty(TaggerAPIConfigurationProperty.STATUS_CODE_SUCCESS), "Found 0 human labeled documents")).build(); From 1d720ec92ee7e32d3225ffa2f0de3daf11f1504b Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Tue, 4 Aug 2015 16:49:29 +0530 Subject: [PATCH 43/44] Fixed csv download link null issue if there wasn't any tweet. --- .../src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java index 4e2a56d2d..22649f368 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java @@ -709,6 +709,11 @@ public int compare(File f1, File f2) { writer = csv.writeClassifiedTweetsCSV(runningHeader, tweetsList.subList(0, countToWrite), collectionCode, fileName, writer); tweetsList.clear(); } + //In case there wasn't any tweet. Just create an empty csv file. + if(countToWrite==0 && tweetsList.size()==0 && 0 == currentSize && runningHeader == null && writer == null ){ + runningHeader = csv.resetClassifiedTweetHeader(ReadWriteCSV.ClassifiedTweetCSVHeader, ReadWriteCSV.FIXED_CLASSIFIED_TWEET_HEADER_SIZE, 0); + writer = csv.writeClassifiedTweetsCSV(runningHeader, tweetsList.subList(0, countToWrite), collectionCode, fileName, writer); + } } } catch (FileNotFoundException ex) { logger.error(collectionCode + ": couldn't find file"); From 3edfe483422b3f9e6ad0600b5951a9562929291a Mon Sep 17 00:00:00 2001 From: Kushal Kant Goyal Date: Tue, 4 Aug 2015 17:13:44 +0530 Subject: [PATCH 44/44] Fixed csv download link null issue for tweetIds if there wasn't any tweet. --- .../src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java index 22649f368..2d2720e71 100644 --- a/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java +++ b/aidr-persister/src/main/java/qa/qcri/aidr/utils/JsonDeserializer.java @@ -353,6 +353,11 @@ public Map generateClassifiedJson2TweetIdsCSVFiltered(final Stri totalCount += countToWrite; tweetsList.clear(); } + //In case if there wasn't any tweet. Just create an empty csv file. + if(countToWrite == 0 && tweetsList.isEmpty() && 0 == totalCount && runningHeader == null && writer == null){ + runningHeader = csv.resetClassifiedTweetHeader(ReadWriteCSV.ClassifiedTweetIDCSVHeader, ReadWriteCSV.FIXED_CLASSIFIED_TWEET_ID_HEADER_SIZE, 0); + writer = csv.writeClassifiedTweetIDsCSV(runningHeader, writer, tweetsList, collectionCode, fileName); + } } finally { if (writer != null) { try {