From 854f9aecd895efbbb7c5be63f3f11c09b01e0c1b Mon Sep 17 00:00:00 2001 From: 3bst0r Date: Wed, 22 Sep 2021 16:37:46 +0200 Subject: [PATCH 1/2] code format, maven compiler plugin --- core/pom.xml | 10 + couchbase2/pom.xml | 12 + .../ycsb/db/couchbase2/Couchbase2Client.java | 247 +++++++++--------- .../java/com/yahoo/ycsb/db/MongoDbClient.java | 58 ++-- 4 files changed, 173 insertions(+), 154 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 44ed12a69d..a4fd26dbd3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -77,5 +77,15 @@ LICENSE file. true + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + diff --git a/couchbase2/pom.xml b/couchbase2/pom.xml index fcdc20d2f3..7b60951fb8 100644 --- a/couchbase2/pom.xml +++ b/couchbase2/pom.xml @@ -20,6 +20,18 @@ LICENSE file. xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + com.yahoo.ycsb binding-parent diff --git a/couchbase2/src/main/java/com/yahoo/ycsb/db/couchbase2/Couchbase2Client.java b/couchbase2/src/main/java/com/yahoo/ycsb/db/couchbase2/Couchbase2Client.java index f550bc538f..db773943ec 100644 --- a/couchbase2/src/main/java/com/yahoo/ycsb/db/couchbase2/Couchbase2Client.java +++ b/couchbase2/src/main/java/com/yahoo/ycsb/db/couchbase2/Couchbase2Client.java @@ -138,7 +138,7 @@ public class Couchbase2Client extends DB { private String soeReadN1qlQuery; private int documentExpiry; private Boolean isSOETest; - + @Override public void init() throws DBException { Properties props = getProperties(); @@ -162,8 +162,8 @@ public void init() throws DBException { runtimeMetricsInterval = Integer.parseInt(props.getProperty("couchbase.runtimeMetricsInterval", "0")); documentExpiry = Integer.parseInt(props.getProperty("couchbase.documentExpiry", "0")); isSOETest = props.getProperty("couchbase.soe", "false").equals("true"); - scanAllQuery = "SELECT RAW meta().id FROM `" + bucketName + - "` WHERE meta().id >= $1 ORDER BY meta().id LIMIT $2"; + scanAllQuery = "SELECT RAW meta().id FROM `" + bucketName + + "` WHERE meta().id >= $1 ORDER BY meta().id LIMIT $2"; soeQuerySelectIDClause = "SELECT RAW meta().id FROM"; soeQuerySelectAllClause = "SELECT RAW `" + bucketName + "` FROM "; @@ -174,10 +174,10 @@ public void init() throws DBException { soeInsertN1qlQuery = "INSERT INTO `" + bucketName + "`(KEY,VALUE) VALUES ($1,$2)"; - soeScanN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + + soeScanN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + "` WHERE meta().id >= $1 ORDER BY meta().id LIMIT $2"; - soeScanKVQuery = soeQuerySelectIDClause + " `" + bucketName + + soeScanKVQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE meta().id >= $1 ORDER BY meta().id LIMIT $2"; try { @@ -187,10 +187,10 @@ public void init() throws DBException { LatencyMetricsCollectorConfig latencyConfig = networkMetricsInterval <= 0 ? DefaultLatencyMetricsCollectorConfig.disabled() : DefaultLatencyMetricsCollectorConfig - .builder() - .emitFrequency(networkMetricsInterval) - .emitFrequencyUnit(TimeUnit.SECONDS) - .build(); + .builder() + .emitFrequency(networkMetricsInterval) + .emitFrequencyUnit(TimeUnit.SECONDS) + .build(); MetricsCollectorConfig runtimeConfig = runtimeMetricsInterval <= 0 ? DefaultMetricsCollectorConfig.disabled() @@ -279,7 +279,7 @@ public Status soeLoad(String table, Generator generator) { try { JsonNode json = JacksonTransformers.MAPPER.readTree(doc.content()); - for (Iterator> jsonFields = json.fields(); jsonFields.hasNext();) { + for (Iterator> jsonFields = json.fields(); jsonFields.hasNext(); ) { Map.Entry jsonField = jsonFields.next(); String name = jsonField.getKey(); if (name.equals(Generator.SOE_FIELD_CUSTOMER_ORDER_LIST)) { @@ -290,7 +290,7 @@ public Status soeLoad(String table, Generator generator) { } if (orders.size() > 0) { String pickedOrder; - if (orders.size() >1) { + if (orders.size() > 1) { Collections.shuffle(orders); } pickedOrder = orders.get(0); @@ -318,7 +318,7 @@ public Status soeLoad(String table, Generator generator) { // ********************* SOE Insert ******************************** @Override - public Status soeInsert(String table, HashMap result, Generator gen) { + public Status soeInsert(String table, HashMap result, Generator gen) { try { //Pair inserDocPair = gen.getInsertDocument(); @@ -336,7 +336,7 @@ public Status soeInsert(String table, HashMap result, Gene private Status soeInsertKv(Generator gen) { int tries = 60; // roughly 60 seconds with the 1 second sleep, not 100% accurate. - for(int i = 0; i < tries; i++) { + for (int i = 0; i < tries; i++) { try { waitForMutationResponse(bucket.async().insert( RawJsonDocument.create(gen.getPredicate().getDocid(), documentExpiry, gen.getPredicate().getValueA()), @@ -376,7 +376,7 @@ private Status soeInsertN1ql(Generator gen) // ********************* SOE Update ******************************** @Override - public Status soeUpdate(String table, HashMap result, Generator gen) { + public Status soeUpdate(String table, HashMap result, Generator gen) { try { if (kv) { return soeUpdateKv(gen); @@ -389,7 +389,7 @@ public Status soeUpdate(String table, HashMap result, Gene } } - private Status soeUpdateKv(Generator gen) { + private Status soeUpdateKv(Generator gen) { waitForMutationResponse(bucket.async().replace( RawJsonDocument.create(gen.getCustomerIdWithDistribution(), documentExpiry, gen.getPredicate().getValueA()), @@ -400,8 +400,7 @@ private Status soeUpdateKv(Generator gen) { return Status.OK; } - private Status soeUpdateN1ql(Generator gen) - throws Exception { + private Status soeUpdateN1ql(Generator gen) { String updateQuery = "UPDATE `" + bucketName + "` USE KEYS [$1] SET " + gen.getPredicate().getNestedPredicateA().getName() + " = $2"; @@ -418,7 +417,7 @@ private Status soeUpdateN1ql(Generator gen) } -// ********************* SOE Read ******************************** + // ********************* SOE Read ******************************** @Override public Status soeRead(String table, HashMap result, Generator gen) { try { @@ -523,7 +522,7 @@ public Observable call(AsyncN1qlQueryResult result) { @Override public Observable call(AsyncN1qlQueryRow row) { String id = new String(row.byteValue()).trim(); - return bucket.async().get(id.substring(1, id.length()-1), RawJsonDocument.class); + return bucket.async().get(id.substring(1, id.length() - 1), RawJsonDocument.class); } }) .map(new Func1>() { @@ -593,7 +592,7 @@ private Status soeSearchKv(final Vector> result, G int offset = gen.getRandomOffset(); final List> data = new ArrayList>(recordcount); - String soeSearchKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE " + + String soeSearchKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE " + gen.getPredicatesSequence().get(0).getName() + "." + gen.getPredicatesSequence().get(0).getNestedPredicateA().getName() + "= $1 AND " + gen.getPredicatesSequence().get(1).getName() + " = $2 AND DATE_PART_STR(" + @@ -630,7 +629,7 @@ public Observable call(AsyncN1qlQueryResult result) { @Override public Observable call(AsyncN1qlQueryRow row) { String id = new String(row.byteValue()).trim(); - return bucket.async().get(id.substring(1, id.length()-1), RawJsonDocument.class); + return bucket.async().get(id.substring(1, id.length() - 1), RawJsonDocument.class); } }) .map(new Func1>() { @@ -657,7 +656,7 @@ private Status soeSearchN1ql(final Vector> result, int recordcount = gen.getRandomLimit(); int offset = gen.getRandomOffset(); - String soeSearchN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + "` WHERE " + + String soeSearchN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + "` WHERE " + gen.getPredicatesSequence().get(0).getName() + "." + gen.getPredicatesSequence().get(0).getNestedPredicateA().getName() + "= $1 AND " + gen.getPredicatesSequence().get(1).getName() + " = $2 AND DATE_PART_STR(" + @@ -712,7 +711,7 @@ private Status soePageKv(final Vector> result, Gen int offset = gen.getRandomOffset(); final List> data = new ArrayList>(recordcount); - String soePageKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE " + gen.getPredicate().getName() + + String soePageKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE " + gen.getPredicate().getName() + "." + gen.getPredicate().getNestedPredicateA().getName() + " = $1 OFFSET $2 LIMIT $3"; bucket.async() @@ -740,7 +739,7 @@ public Observable call(AsyncN1qlQueryResult result) { @Override public Observable call(AsyncN1qlQueryRow row) { String id = new String(row.byteValue()).trim(); - return bucket.async().get(id.substring(1, id.length()-1), RawJsonDocument.class); + return bucket.async().get(id.substring(1, id.length() - 1), RawJsonDocument.class); } }) .map(new Func1>() { @@ -770,7 +769,7 @@ private Status soePageN1ql(final Vector> result, G int recordcount = gen.getRandomLimit(); int offset = gen.getRandomOffset(); - String soePageN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + "` WHERE " + + String soePageN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + "` WHERE " + gen.getPredicate().getName() + "." + gen.getPredicate().getNestedPredicateA().getName() + " = $1 OFFSET $2 LIMIT $3"; @@ -815,7 +814,7 @@ private Status soeNestScanKv(final Vector> result, int recordcount = gen.getRandomLimit(); final List> data = new ArrayList>(recordcount); - String soeNestScanKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE " + + String soeNestScanKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE " + gen.getPredicate().getName() + "." + gen.getPredicate().getNestedPredicateA().getName() + "." + gen.getPredicate().getNestedPredicateA().getNestedPredicateA().getName() + " = $1 LIMIT $2"; @@ -823,7 +822,7 @@ private Status soeNestScanKv(final Vector> result, bucket.async() .query(N1qlQuery.parameterized( soeNestScanKvQuery, - JsonArray.from(gen.getPredicate().getNestedPredicateA().getNestedPredicateA().getValueA(), recordcount), + JsonArray.from(gen.getPredicate().getNestedPredicateA().getNestedPredicateA().getValueA(), recordcount), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )) .doOnNext(new Action1() { @@ -845,7 +844,7 @@ public Observable call(AsyncN1qlQueryResult result) { @Override public Observable call(AsyncN1qlQueryRow row) { String id = new String(row.byteValue()).trim(); - return bucket.async().get(id.substring(1, id.length()-1), RawJsonDocument.class); + return bucket.async().get(id.substring(1, id.length() - 1), RawJsonDocument.class); } }) .map(new Func1>() { @@ -871,14 +870,14 @@ public void call(HashMap tuple) { private Status soeNestScanN1ql(final Vector> result, Generator gen) { int recordcount = gen.getRandomLimit(); - String soeNestScanN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + "` WHERE " + + String soeNestScanN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + "` WHERE " + gen.getPredicate().getName() + "." + gen.getPredicate().getNestedPredicateA().getName() + "." + gen.getPredicate().getNestedPredicateA().getNestedPredicateA().getName() + " = $1 LIMIT $2"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( soeNestScanN1qlQuery, - JsonArray.from(gen.getPredicate().getNestedPredicateA().getNestedPredicateA().getValueA(), recordcount), + JsonArray.from(gen.getPredicate().getNestedPredicateA().getNestedPredicateA().getValueA(), recordcount), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )); @@ -918,13 +917,13 @@ private Status soeArrayScanKv(final Vector> result int recordcount = gen.getRandomLimit(); final List> data = new ArrayList>(recordcount); - String soeArrayScanKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE ANY v IN " + + String soeArrayScanKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE ANY v IN " + gen.getPredicate().getName() + " SATISFIES v = $1 END ORDER BY meta().id LIMIT $2"; bucket.async() .query(N1qlQuery.parameterized( soeArrayScanKvQuery, - JsonArray.from(gen.getPredicate().getValueA(), recordcount), + JsonArray.from(gen.getPredicate().getValueA(), recordcount), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )) .doOnNext(new Action1() { @@ -946,7 +945,7 @@ public Observable call(AsyncN1qlQueryResult result) { @Override public Observable call(AsyncN1qlQueryRow row) { String id = new String(row.byteValue()).trim(); - return bucket.async().get(id.substring(1, id.length()-1), RawJsonDocument.class); + return bucket.async().get(id.substring(1, id.length() - 1), RawJsonDocument.class); } }) .map(new Func1>() { @@ -973,12 +972,12 @@ public void call(HashMap tuple) { private Status soeArrayScanN1ql(final Vector> result, Generator gen) { int recordcount = gen.getRandomLimit(); - String soeArrayScanN1qlQuery = soeQuerySelectAllClause + "`" + bucketName + "` WHERE ANY v IN " + + String soeArrayScanN1qlQuery = soeQuerySelectAllClause + "`" + bucketName + "` WHERE ANY v IN " + gen.getPredicate().getName() + " SATISFIES v = $1 END ORDER BY meta().id LIMIT $2"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( soeArrayScanN1qlQuery, - JsonArray.from(gen.getPredicate().getValueA(), recordcount), + JsonArray.from(gen.getPredicate().getValueA(), recordcount), N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )); @@ -1025,7 +1024,7 @@ private Status soeArrayDeepScanKv(final Vector> re String cityCountryValue = gen.getPredicate().getNestedPredicateA().getValueA() + "." + gen.getPredicate().getNestedPredicateB().getValueA(); - String soeArrayDeepScanKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE ANY v IN " + String soeArrayDeepScanKvQuery = soeQuerySelectIDClause + " `" + bucketName + "` WHERE ANY v IN " + visitedPlacesFieldName + " SATISFIES ANY c IN v." + cityFieldName + " SATISFIES (v." + countryFieldName + " || \".\" || c) = $1 END END ORDER BY META().id LIMIT $2"; @@ -1054,7 +1053,7 @@ public Observable call(AsyncN1qlQueryResult result) { @Override public Observable call(AsyncN1qlQueryRow row) { String id = new String(row.byteValue()).trim(); - return bucket.async().get(id.substring(1, id.length()-1), RawJsonDocument.class); + return bucket.async().get(id.substring(1, id.length() - 1), RawJsonDocument.class); } }) .map(new Func1>() { @@ -1088,7 +1087,7 @@ private Status soeArrayDeepScanN1ql(final Vector> String cityCountryValue = gen.getPredicate().getNestedPredicateA().getValueA() + "." + gen.getPredicate().getNestedPredicateB().getValueA(); - String soeArrayDeepScanN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + "` WHERE ANY v IN " + String soeArrayDeepScanN1qlQuery = soeQuerySelectAllClause + " `" + bucketName + "` WHERE ANY v IN " + visitedPlacesFieldName + " SATISFIES ANY c IN v." + cityFieldName + " SATISFIES (v." + countryFieldName + " || \".\" || c) = $1 END END ORDER BY META().id LIMIT $2"; @@ -1136,15 +1135,15 @@ private Status soeReport1Kv(final Vector> result, private Status soeReport1N1ql(final Vector> result, Generator gen) { - String soeReport1N1qlQuery = "SELECT * FROM `" + bucketName + "` c1 INNER JOIN `" + + String soeReport1N1qlQuery = "SELECT * FROM `" + bucketName + "` c1 INNER JOIN `" + bucketName + "` o1 ON KEYS c1." + gen.getPredicatesSequence().get(0).getName() + " WHERE c1." + gen.getPredicatesSequence().get(1).getName() + "." + - gen.getPredicatesSequence().get(1).getNestedPredicateA().getName()+ " = $1 "; + gen.getPredicatesSequence().get(1).getNestedPredicateA().getName() + " = $1 "; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( - soeReport1N1qlQuery, - JsonArray.from(gen.getPredicatesSequence().get(1).getNestedPredicateA().getValueA()), - N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) + soeReport1N1qlQuery, + JsonArray.from(gen.getPredicatesSequence().get(1).getNestedPredicateA().getValueA()), + N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )); if (!queryResult.parseSuccess() || !queryResult.finalSuccess()) { throw new RuntimeException("Error while parsing N1QL Result. Query: " + soeReport1N1qlQuery @@ -1185,14 +1184,14 @@ private Status soeReport2N1ql(final Vector> result String nameOrderMonth = gen.getPredicatesSequence().get(0).getName(); String nameOrderSaleprice = gen.getPredicatesSequence().get(1).getName(); - String nameAddress = gen.getPredicatesSequence().get(2).getName(); - String nameAddressZip = gen.getPredicatesSequence().get(2).getNestedPredicateA().getName(); + String nameAddress = gen.getPredicatesSequence().get(2).getName(); + String nameAddressZip = gen.getPredicatesSequence().get(2).getNestedPredicateA().getName(); String nameOrderlist = gen.getPredicatesSequence().get(3).getName(); String valueOrderMonth = gen.getPredicatesSequence().get(0).getValueA(); - String valueAddressZip = gen.getPredicatesSequence().get(2).getNestedPredicateA().getValueA(); + String valueAddressZip = gen.getPredicatesSequence().get(2).getNestedPredicateA().getValueA(); String soeReport2N1qlQuery = "SELECT o2." + nameOrderMonth + ", c2." + nameAddress + "." + nameAddressZip + - ", SUM(o2." + nameOrderSaleprice + ") FROM `" + bucketName + "` c2 INNER JOIN `" + bucketName + + ", SUM(o2." + nameOrderSaleprice + ") FROM `" + bucketName + "` c2 INNER JOIN `" + bucketName + "` o2 ON KEYS c2." + nameOrderlist + " WHERE c2." + nameAddress + "." + nameAddressZip + " = $1 AND o2." + nameOrderMonth + " = $2 GROUP BY o2." + nameOrderMonth + ", c2." + nameAddress + "." + nameAddressZip + " ORDER BY SUM(o2." + nameOrderSaleprice + ")"; @@ -1218,10 +1217,9 @@ private Status soeReport2N1ql(final Vector> result // ************************************************************************************************ - @Override public Status read(final String table, final String key, Set fields, - final HashMap result) { + final HashMap result) { try { String docId = formatId(table, key); if (kv) { @@ -1239,13 +1237,13 @@ public Status read(final String table, final String key, Set fields, /** * Performs the {@link #read(String, String, Set, HashMap)} operation via Key/Value ("get"). * - * @param docId the document ID + * @param docId the document ID * @param fields the fields to be loaded * @param result the result map where the doc needs to be converted into * @return The result of the operation. */ private Status readKv(final String docId, final Set fields, final HashMap result) - throws Exception { + throws Exception { RawJsonDocument loaded = bucket.get(docId, RawJsonDocument.class); if (loaded == null) { return Status.NOT_FOUND; @@ -1256,16 +1254,16 @@ private Status readKv(final String docId, final Set fields, final HashMa /** * Performs the {@link #read(String, String, Set, HashMap)} operation via N1QL ("SELECT"). - * + *

* If this option should be used, the "-p couchbase.kv=false" property must be set. * - * @param docId the document ID + * @param docId the document ID * @param fields the fields to be loaded * @param result the result map where the doc needs to be converted into * @return The result of the operation. */ private Status readN1ql(final String docId, Set fields, final HashMap result) - throws Exception { + throws Exception { String readQuery = "SELECT " + joinFields(fields) + " FROM `" + bucketName + "` USE KEYS [$1]"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( readQuery, @@ -1275,7 +1273,7 @@ private Status readN1ql(final String docId, Set fields, final HashMap /** * Performs the {@link #update(String, String, HashMap)} operation via N1QL ("UPDATE"). - * + *

* If this option should be used, the "-p couchbase.kv=false" property must be set. * - * @param docId the document ID + * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status updateN1ql(final String docId, final HashMap values) - throws Exception { + throws Exception { String fields = encodeN1qlFields(values); String updateQuery = "UPDATE `" + bucketName + "` USE KEYS [$1] SET " + fields; @@ -1357,7 +1355,7 @@ private Status updateN1ql(final String docId, final HashMap * Note that during the "load" phase it makes sense to retry TMPFAILS (so that even if the server is * overloaded temporarily the ops will succeed eventually). The current code will retry TMPFAILs * for maximum of one minute and then bubble up the error. * - * @param docId the document ID + * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status insertKv(final String docId, final HashMap values) { int tries = 60; // roughly 60 seconds with the 1 second sleep, not 100% accurate. - for(int i = 0; i < tries; i++) { + for (int i = 0; i < tries; i++) { try { waitForMutationResponse(bucket.async().insert( RawJsonDocument.create(docId, documentExpiry, encode(values)), @@ -1415,20 +1413,20 @@ private Status insertKv(final String docId, final HashMap } throw new RuntimeException("Still receiving TMPFAIL from the server after trying " + tries + " times. " + - "Check your server."); + "Check your server."); } /** * Performs the {@link #insert(String, String, HashMap)} operation via N1QL ("INSERT"). - * + *

* If this option should be used, the "-p couchbase.kv=false" property must be set. * - * @param docId the document ID + * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status insertN1ql(final String docId, final HashMap values) - throws Exception { + throws Exception { String insertQuery = "INSERT INTO `" + bucketName + "`(KEY,VALUE) VALUES ($1,$2)"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( @@ -1439,18 +1437,18 @@ private Status insertN1ql(final String docId, final HashMap * If this option should be used, the "-p couchbase.upsert=true" property must be set. * - * @param table The name of the table - * @param key The record key of the record to insert. + * @param table The name of the table + * @param key The record key of the record to insert. * @param values A HashMap of field/value pairs to insert in the record * @return The result of the operation. */ @@ -1470,10 +1468,10 @@ private Status upsert(final String table, final String key, final HashMap * If this option should be used, the "-p couchbase.upsert=true" property must be set. * - * @param docId the document ID + * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ @@ -1488,15 +1486,15 @@ private Status upsertKv(final String docId, final HashMap /** * Performs the {@link #upsert(String, String, HashMap)} operation via N1QL ("UPSERT"). - * + *

* If this option should be used, the "-p couchbase.upsert=true -p couchbase.kv=false" properties must be set. * - * @param docId the document ID + * @param docId the document ID * @param values the values to update the document with. * @return The result of the operation. */ private Status upsertN1ql(final String docId, final HashMap values) - throws Exception { + throws Exception { String upsertQuery = "UPSERT INTO `" + bucketName + "`(KEY,VALUE) VALUES ($1,$2)"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( @@ -1507,7 +1505,7 @@ private Status upsertN1ql(final String docId, final HashMap * If this option should be used, the "-p couchbase.kv=false" property must be set. * * @param docId the document ID. @@ -1560,14 +1558,14 @@ private Status deleteN1ql(final String docId) throws Exception { if (!queryResult.parseSuccess() || !queryResult.finalSuccess()) { throw new DBException("Error while parsing N1QL Result. Query: " + deleteQuery - + ", Errors: " + queryResult.errors()); + + ", Errors: " + queryResult.errors()); } return Status.OK; } @Override public Status scan(final String table, final String startkey, final int recordcount, final Set fields, - final Vector> result) { + final Vector> result) { try { if (fields == null || fields.isEmpty()) { return scanAllFields(table, startkey, recordcount, result); @@ -1582,25 +1580,25 @@ public Status scan(final String table, final String startkey, final int recordco /** * Performs the {@link #scan(String, String, int, Set, Vector)} operation, optimized for all fields. - * + *

* Since the full document bodies need to be loaded anyways, it makes sense to just grab the document IDs * from N1QL and then perform the bulk loading via KV for better performance. This is a usual pattern with * Couchbase and shows the benefits of using both N1QL and KV together. * - * @param table The name of the table - * @param startkey The record key of the first record to read. + * @param table The name of the table + * @param startkey The record key of the first record to read. * @param recordcount The number of records to read - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record + * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return The result of the operation. */ private Status scanAllFields(final String table, final String startkey, final int recordcount, - final Vector> result) { + final Vector> result) { final List> data = new ArrayList>(recordcount); bucket.async() .query(N1qlQuery.parameterized( - scanAllQuery, - JsonArray.from(formatId(table, startkey), recordcount), - N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) + scanAllQuery, + JsonArray.from(formatId(table, startkey), recordcount), + N1qlParams.build().adhoc(adhoc).maxParallelism(maxParallelism) )) .doOnNext(new Action1() { @Override @@ -1608,7 +1606,7 @@ public void call(AsyncN1qlQueryResult result) { if (!result.parseSuccess()) { throw new RuntimeException("Error while parsing N1QL Result. Query: " + scanAllQuery - + ", Errors: " + result.errors()); + + ", Errors: " + result.errors()); } } }) @@ -1622,7 +1620,7 @@ public Observable call(AsyncN1qlQueryResult result) { @Override public Observable call(AsyncN1qlQueryRow row) { String id = new String(row.byteValue()).trim(); - return bucket.async().get(id.substring(1, id.length()-1), RawJsonDocument.class); + return bucket.async().get(id.substring(1, id.length() - 1), RawJsonDocument.class); } }) .map(new Func1>() { @@ -1648,15 +1646,15 @@ public void call(HashMap tuple) { /** * Performs the {@link #scan(String, String, int, Set, Vector)} operation N1Ql only for a subset of the fields. * - * @param table The name of the table - * @param startkey The record key of the first record to read. + * @param table The name of the table + * @param startkey The record key of the first record to read. * @param recordcount The number of records to read - * @param fields The list of fields to read, or null for all of them - * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record + * @param fields The list of fields to read, or null for all of them + * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record * @return The result of the operation. */ private Status scanSpecificFields(final String table, final String startkey, final int recordcount, - final Set fields, final Vector> result) { + final Set fields, final Vector> result) { String scanSpecQuery = "SELECT " + joinFields(fields) + " FROM `" + bucketName + "` WHERE meta().id >= $1 LIMIT $2"; N1qlQueryResult queryResult = bucket.query(N1qlQuery.parameterized( @@ -1667,7 +1665,7 @@ private Status scanSpecificFields(final String table, final String startkey, fin if (!queryResult.parseSuccess() || !queryResult.finalSuccess()) { throw new RuntimeException("Error while parsing N1QL Result. Query: " + scanSpecQuery - + ", Errors: " + queryResult.errors()); + + ", Errors: " + queryResult.errors()); } boolean allFields = fields == null || fields.isEmpty(); @@ -1690,7 +1688,7 @@ private Status scanSpecificFields(final String table, final String startkey, fin /** * Helper method to block on the response, depending on the property set. - * + *

* By default, since YCSB is sync the code will always wait for the operation to complete. In some * cases it can be useful to just "drive load" and disable the waiting. Note that when the * "-p couchbase.syncMutationResponse=false" option is used, the measured results by YCSB can basically @@ -1700,7 +1698,7 @@ private Status scanSpecificFields(final String table, final String startkey, fin */ private void waitForMutationResponse(final Observable> input) { if (!syncMutResponse) { - ((Observable>)input).subscribe(new Subscriber>() { + ((Observable>) input).subscribe(new Subscriber>() { @Override public void onCompleted() { } @@ -1775,7 +1773,7 @@ private static String joinFields(final Set fields) { * Helper method to turn the prefix and key into a proper document ID. * * @param prefix the prefix (table). - * @param key the key itself. + * @param key the key itself. * @return a document ID that can be used with Couchbase. */ private static String formatId(final String prefix, final String key) { @@ -1792,16 +1790,16 @@ private static ReplicateTo parseReplicateTo(final String property) throws DBExce int value = Integer.parseInt(property); switch (value) { - case 0: - return ReplicateTo.NONE; - case 1: - return ReplicateTo.ONE; - case 2: - return ReplicateTo.TWO; - case 3: - return ReplicateTo.THREE; - default: - throw new DBException("\"couchbase.replicateTo\" must be between 0 and 3"); + case 0: + return ReplicateTo.NONE; + case 1: + return ReplicateTo.ONE; + case 2: + return ReplicateTo.TWO; + case 3: + return ReplicateTo.THREE; + default: + throw new DBException("\"couchbase.replicateTo\" must be between 0 and 3"); } } @@ -1815,18 +1813,18 @@ private static PersistTo parsePersistTo(final String property) throws DBExceptio int value = Integer.parseInt(property); switch (value) { - case 0: - return PersistTo.NONE; - case 1: - return PersistTo.ONE; - case 2: - return PersistTo.TWO; - case 3: - return PersistTo.THREE; - case 4: - return PersistTo.FOUR; - default: - throw new DBException("\"couchbase.persistTo\" must be between 0 and 4"); + case 0: + return PersistTo.NONE; + case 1: + return PersistTo.ONE; + case 2: + return PersistTo.TWO; + case 3: + return PersistTo.THREE; + case 4: + return PersistTo.FOUR; + default: + throw new DBException("\"couchbase.persistTo\" must be between 0 and 4"); } } @@ -1835,14 +1833,14 @@ private static PersistTo parsePersistTo(final String property) throws DBExceptio * * @param source the loaded object. * @param fields the fields to check. - * @param dest the result passed back to YCSB. + * @param dest the result passed back to YCSB. */ private void decode(final String source, final Set fields, final HashMap dest) { try { JsonNode json = JacksonTransformers.MAPPER.readTree(source); boolean checkFields = fields != null && !fields.isEmpty(); - for (Iterator> jsonFields = json.fields(); jsonFields.hasNext();) { + for (Iterator> jsonFields = json.fields(); jsonFields.hasNext(); ) { Map.Entry jsonField = jsonFields.next(); String name = jsonField.getKey(); if (checkFields && !fields.contains(name)) { @@ -1883,6 +1881,7 @@ private String encode(final HashMap source) { /** * handling rich JSON types by converting Json arrays and Json objects into String. + * * @param source * @param fields * @param dest @@ -1892,7 +1891,7 @@ private void soeDecode(final String source, final Set fields, try { JsonNode json = JacksonTransformers.MAPPER.readTree(source); boolean checkFields = fields != null && !fields.isEmpty(); - for (Iterator> jsonFields = json.fields(); jsonFields.hasNext();) { + for (Iterator> jsonFields = json.fields(); jsonFields.hasNext(); ) { Map.Entry jsonField = jsonFields.next(); String name = jsonField.getKey(); if (checkFields && !fields.contains(name)) { diff --git a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java index 22e4207627..8da9ce6289 100644 --- a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java +++ b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java @@ -1,12 +1,12 @@ /** * Copyright (c) 2012 - 2015 YCSB contributors. All rights reserved. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You * may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or @@ -59,7 +59,7 @@ *

* See the README.md for configuration information. *

- * + * * @author ypai * @see MongoDB Inc. * driver @@ -157,7 +157,7 @@ public Status soeLoad(String table, Generator generator) { generator.putCustomerDocument(key, queryResult.toJson()); List orders = (List) queryResult.get(Generator.SOE_FIELD_CUSTOMER_ORDER_LIST); - for (String order:orders) { + for (String order : orders) { query = new Document("_id", order); findIterable = collection.find(query); queryResult = findIterable.first(); @@ -177,7 +177,7 @@ public Status soeLoad(String table, Generator generator) { // ********************* SOE Insert ******************************** @Override - public Status soeInsert(String table, HashMap result, Generator gen) { + public Status soeInsert(String table, HashMap result, Generator gen) { try { MongoCollection collection = database.getCollection(table); @@ -199,7 +199,6 @@ public Status soeInsert(String table, HashMap result, Gene } - // ********************* SOE Update ******************************** @Override @@ -387,7 +386,7 @@ public Status soeSearch(String table, final Vector new BasicDBObject("$gte", new SimpleDateFormat("yyyy-MM-dd").parse(dobyearValue + "-1-1")); DBObject clause3 = new BasicDBObject(dobyearName, clause3Range); DBObject clause4Range = - new BasicDBObject("$lte", new SimpleDateFormat("yyyy-MM-dd").parse(dobyearValue+ "-12-31")); + new BasicDBObject("$lte", new SimpleDateFormat("yyyy-MM-dd").parse(dobyearValue + "-12-31")); DBObject clause4 = new BasicDBObject(dobyearName, clause4Range); BasicDBList and = new BasicDBList(); @@ -441,7 +440,7 @@ public Status soeSearch(String table, final Vector @Override public Status soeNestScan(String table, final Vector> result, Generator gen) { int recordcount = gen.getRandomLimit(); - String nestedZipName = gen.getPredicate().getName() + "." + gen.getPredicate().getNestedPredicateA().getName() + + String nestedZipName = gen.getPredicate().getName() + "." + gen.getPredicate().getNestedPredicateA().getName() + "." + gen.getPredicate().getNestedPredicateA().getNestedPredicateA().getName(); String nestedZipValue = gen.getPredicate().getNestedPredicateA().getNestedPredicateA().getValueA(); @@ -492,7 +491,7 @@ public Status soeNestScan(String table, final Vector> result, Generator gen) { int recordcount = gen.getRandomLimit(); - String arrName = gen.getPredicate().getName(); + String arrName = gen.getPredicate().getName(); String arrValue = gen.getPredicate().getValueA(); Document sort = new Document("_id", INCLUDE); @@ -500,7 +499,7 @@ public Status soeArrayScan(String table, final Vector collection = database.getCollection(table); - BasicDBObject query = new BasicDBObject(); + BasicDBObject query = new BasicDBObject(); query.put(arrName, arrValue); FindIterable findIterable = collection.find(query).sort(sort).limit(recordcount); @@ -539,7 +538,7 @@ public Status soeArrayScan(String table, final Vector> result, Generator gen) { int recordcount = gen.getRandomLimit(); - String fieldName = gen.getPredicate().getName(); + String fieldName = gen.getPredicate().getName(); String fieldCountryName = gen.getPredicate().getNestedPredicateA().getName(); String fieldCitiesName = gen.getPredicate().getNestedPredicateB().getName(); String fieldCountryValue = gen.getPredicate().getNestedPredicateA().getValueA(); @@ -551,7 +550,7 @@ public Status soeArrayDeepScan(String table, final Vector collection = database.getCollection(table); - BasicDBObject query = new BasicDBObject(); + BasicDBObject query = new BasicDBObject(); query.put(fieldName + "." + fieldCountryName, fieldCountryValue); query.put(fieldName + "." + fieldCitiesName, fieldCitiesValue); @@ -587,7 +586,6 @@ public Status soeArrayDeepScan(String table, final Vector HashMap resultMap = new HashMap(); Document obj = cursor.next(); if (obj.get(orderListName) != null) { - BasicDBObject subq = new BasicDBObject(); + BasicDBObject subq = new BasicDBObject(); subq.put("_id", new BasicDBObject("$in", obj.get(orderListName))); FindIterable findSubIterable = collection.find(subq); Document orderDoc = findSubIterable.first(); @@ -647,11 +645,11 @@ public Status soeReport2(String table, final Vector cursor = null; try { @@ -673,7 +671,7 @@ public Status soeReport2(String table, final Vector values) { + HashMap values) { try { MongoCollection collection = database.getCollection(table); Document toInsert = new Document("_id", key); @@ -855,7 +853,7 @@ public Status insert(String table, String key, bulkInserts.add(toInsert); if (bulkInserts.size() == batchSize) { if (useUpsert) { - List> updates = + List> updates = new ArrayList>(bulkInserts.size()); for (Document doc : bulkInserts) { updates.add(new UpdateOneModel( @@ -884,7 +882,7 @@ public Status insert(String table, String key, /** * Read a record from the database. Each field/value pair from the result will * be stored in a HashMap. - * + * * @param table * The name of the table * @param key @@ -897,7 +895,7 @@ public Status insert(String table, String key, */ @Override public Status read(String table, String key, Set fields, - HashMap result) { + HashMap result) { try { MongoCollection collection = database.getCollection(table); Document query = new Document("_id", key); @@ -928,7 +926,7 @@ public Status read(String table, String key, Set fields, /** * Perform a range scan for a set of records in the database. Each field/value * pair from the result will be stored in a HashMap. - * + * * @param table * The name of the table * @param startkey @@ -945,7 +943,7 @@ public Status read(String table, String key, Set fields, */ @Override public Status scan(String table, String startkey, int recordcount, - Set fields, Vector> result) { + Set fields, Vector> result) { MongoCursor cursor = null; try { MongoCollection collection = database.getCollection(table); @@ -999,7 +997,7 @@ public Status scan(String table, String startkey, int recordcount, * Update a record in the database. Any field/value pairs in the specified * values HashMap will be written into the record with the specified record * key, overwriting any existing values with the same field name. - * + * * @param table * The name of the table * @param key @@ -1011,7 +1009,7 @@ public Status scan(String table, String startkey, int recordcount, */ @Override public Status update(String table, String key, - HashMap values) { + HashMap values) { try { MongoCollection collection = database.getCollection(table); @@ -1036,7 +1034,7 @@ public Status update(String table, String key, /** * Fills the map with the values from the DBObject. - * + * * @param resultMap * The map to fill/ * @param obj From d86f5cccb2454f492d475e4edb4f6d26ad788bea Mon Sep 17 00:00:00 2001 From: 3bst0r Date: Wed, 10 Nov 2021 09:10:10 +0100 Subject: [PATCH 2/2] fix incorrect MongoDB soeArrayDeepScan: enforce that matched city is within the visited country. previously would match all customers that visited the country and visited the city. now matches only customers that visited the city within the country. --- .../java/com/yahoo/ycsb/db/MongoDbClient.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java index 8da9ce6289..cb56380856 100644 --- a/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java +++ b/mongodb/src/main/java/com/yahoo/ycsb/db/MongoDbClient.java @@ -46,6 +46,7 @@ import com.yahoo.ycsb.generator.soe.Generator; import org.bson.Document; +import org.bson.conversions.Bson; import org.bson.types.Binary; @@ -550,11 +551,17 @@ public Status soeArrayDeepScan(String table, final Vector collection = database.getCollection(table); - BasicDBObject query = new BasicDBObject(); - query.put(fieldName + "." + fieldCountryName, fieldCountryValue); - query.put(fieldName + "." + fieldCitiesName, fieldCitiesValue); - - FindIterable findIterable = collection.find(query).sort(sort); + final DBObject query = QueryBuilder.start() + .put(fieldName) + .elemMatch(QueryBuilder.start() + .put(fieldCountryName) + .is(fieldCountryValue) + .put(fieldCitiesName) + .is(fieldCitiesValue) + .get()) + .get(); + + FindIterable findIterable = collection.find((Bson) query).sort(sort); Document projection = new Document(); for (String field : gen.getAllFields()) { projection.put(field, INCLUDE); @@ -568,7 +575,7 @@ public Status soeArrayDeepScan(String table, final Vector resultMap = - new HashMap(); + new HashMap<>(); Document obj = cursor.next(); soeFillMap(resultMap, obj); @@ -576,7 +583,7 @@ public Status soeArrayDeepScan(String table, final Vector