From 22372151fb54094cc7cbe025e472a410d88d84b7 Mon Sep 17 00:00:00 2001 From: amaya Date: Tue, 23 Aug 2016 18:19:53 +0900 Subject: [PATCH 01/28] add feature for testing UDF --- pom.xml | 3 +- systemtest/pom.xml | 47 +++ .../java/com/klarna/hiverunner/Extractor.java | 15 + .../main/java/hivemall/ConvertToMsgpack.java | 87 +++++ .../java/hivemall/model/CreateTableHQ.java | 29 ++ .../main/java/hivemall/model/DropTableHQ.java | 7 + .../src/main/java/hivemall/model/HQ.java | 90 +++++ .../main/java/hivemall/model/InsertHQ.java | 52 +++ .../src/main/java/hivemall/model/RawHQ.java | 15 + .../main/java/hivemall/model/StrictHQ.java | 5 + .../src/main/java/hivemall/model/TableHQ.java | 10 + .../main/java/hivemall/model/TableListHQ.java | 5 + .../model/UploadFileAsNewTableHQ.java | 19 + .../java/hivemall/model/UploadFileHQ.java | 38 ++ .../model/UploadFileToExistingHQ.java | 9 + .../model/lazy/LazyMatchingResource.java | 38 ++ .../hivemall/runner/HiveSystemTestRunner.java | 104 ++++++ .../hivemall/runner/SystemTestCommonInfo.java | 18 + .../hivemall/runner/SystemTestRunner.java | 194 ++++++++++ .../java/hivemall/runner/SystemTestTeam.java | 107 ++++++ .../hivemall/runner/TDSystemTestRunner.java | 341 ++++++++++++++++++ .../src/main/java/hivemall/utils/IO.java | 60 +++ 22 files changed, 1292 insertions(+), 1 deletion(-) create mode 100644 systemtest/pom.xml create mode 100644 systemtest/src/main/java/com/klarna/hiverunner/Extractor.java create mode 100644 systemtest/src/main/java/hivemall/ConvertToMsgpack.java create mode 100644 systemtest/src/main/java/hivemall/model/CreateTableHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/DropTableHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/HQ.java create mode 100644 systemtest/src/main/java/hivemall/model/InsertHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/RawHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/StrictHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/TableHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/TableListHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/UploadFileAsNewTableHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/UploadFileHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/UploadFileToExistingHQ.java create mode 100644 systemtest/src/main/java/hivemall/model/lazy/LazyMatchingResource.java create mode 100644 systemtest/src/main/java/hivemall/runner/HiveSystemTestRunner.java create mode 100644 systemtest/src/main/java/hivemall/runner/SystemTestCommonInfo.java create mode 100644 systemtest/src/main/java/hivemall/runner/SystemTestRunner.java create mode 100644 systemtest/src/main/java/hivemall/runner/SystemTestTeam.java create mode 100644 systemtest/src/main/java/hivemall/runner/TDSystemTestRunner.java create mode 100644 systemtest/src/main/java/hivemall/utils/IO.java diff --git a/pom.xml b/pom.xml index 3ad81a46..95174289 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,8 @@ nlp mixserv spark - + systemtest + yyyy diff --git a/systemtest/pom.xml b/systemtest/pom.xml new file mode 100644 index 00000000..d58c64a7 --- /dev/null +++ b/systemtest/pom.xml @@ -0,0 +1,47 @@ + + + + hivemall + io.github.myui + 0.4.2-rc.2 + + 4.0.0 + + systemtest + + + + io.github.myui + hivemall-core + 0.4.2-rc.2 + + + junit + junit + 4.12 + + + com.klarna + hiverunner + 3.0.0 + + + com.treasuredata.client + td-client + 0.7.25 + jar-with-dependencies + + + org.apache.commons + commons-csv + 1.1 + + + org.msgpack + msgpack-core + 0.8.9 + + + diff --git a/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java b/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java new file mode 100644 index 00000000..5ab39089 --- /dev/null +++ b/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java @@ -0,0 +1,15 @@ +package com.klarna.hiverunner; + +import com.klarna.hiverunner.config.HiveRunnerConfig; +import org.junit.rules.TemporaryFolder; + +public class Extractor { + public static StandaloneHiveServerContext getStandaloneHiveServerContext( + TemporaryFolder basedir, HiveRunnerConfig hiveRunnerConfig) { + return new StandaloneHiveServerContext(basedir, hiveRunnerConfig); + } + + public static HiveServerContainer getHiveServerContainer(HiveServerContext context) { + return new HiveServerContainer(context); + } +} diff --git a/systemtest/src/main/java/hivemall/ConvertToMsgpack.java b/systemtest/src/main/java/hivemall/ConvertToMsgpack.java new file mode 100644 index 00000000..adebc3ec --- /dev/null +++ b/systemtest/src/main/java/hivemall/ConvertToMsgpack.java @@ -0,0 +1,87 @@ +package hivemall; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessagePacker; +import org.msgpack.value.ValueFactory; + +import javax.annotation.Nonnull; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.util.List; +import java.util.zip.GZIPOutputStream; + +public class ConvertToMsgpack { + @Nonnull + private final File file; + @Nonnull + private final List header; + @Nonnull + private final CSVFormat format; + + + public ConvertToMsgpack(File file, List header, CSVFormat format) { + this.file = file; + this.header = header; + this.format = format; + } + + + public byte[] asByteArray(final boolean needTimeColumn) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + MessagePacker packer = MessagePack.newDefaultPacker(new GZIPOutputStream(os)); + BufferedReader br = new BufferedReader(new FileReader(file)); + try { + // always skip header, use user-defined or existing table's + CSVParser parser = format.withSkipHeaderRecord().parse(br); + final long time = System.currentTimeMillis() / 1000; + for (CSVRecord record : parser.getRecords()) { + ValueFactory.MapBuilder map = ValueFactory.newMapBuilder(); + + // add `time` column if needed && not exists + if (needTimeColumn && !header.contains("time")) { + map.put(ValueFactory.newString("time"), ValueFactory.newInteger(time)); + } + + // pack each value in row + int i = 0; + for (String val : record) { + map.put(ValueFactory.newString(header.get(i)), ValueFactory.newString(val)); + i++; + } + packer.packValue(map.build()); + } + } finally { + br.close(); + packer.close(); + } + + return os.toByteArray(); + } + + public byte[] asByteArray() throws Exception { + return asByteArray(true); + } + + public File asFile(File to, final boolean needTimeColumn) throws Exception { + FileOutputStream os = null; + try { + os = new FileOutputStream(to); + os.write(asByteArray(needTimeColumn)); + return to; + } finally { + if (os != null) { + os.close(); + } + } + } + + public File asFile(File to) throws Exception { + return asFile(to, true); + } +} diff --git a/systemtest/src/main/java/hivemall/model/CreateTableHQ.java b/systemtest/src/main/java/hivemall/model/CreateTableHQ.java new file mode 100644 index 00000000..7e79f2e8 --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/CreateTableHQ.java @@ -0,0 +1,29 @@ +package hivemall.model; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class CreateTableHQ extends TableHQ { + public LinkedHashMap header; + + + CreateTableHQ(String tableName, LinkedHashMap header) { + super(tableName); + this.header = header; + } + + + public String getTableDeclaration() { + StringBuilder sb = new StringBuilder(); + sb.append("("); + for (Map.Entry e : header.entrySet()) { + sb.append(e.getKey()); + sb.append(" "); + sb.append(e.getValue()); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } +} diff --git a/systemtest/src/main/java/hivemall/model/DropTableHQ.java b/systemtest/src/main/java/hivemall/model/DropTableHQ.java new file mode 100644 index 00000000..31dd2d84 --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/DropTableHQ.java @@ -0,0 +1,7 @@ +package hivemall.model; + +public class DropTableHQ extends TableHQ { + DropTableHQ(String tableName) { + super(tableName); + } +} diff --git a/systemtest/src/main/java/hivemall/model/HQ.java b/systemtest/src/main/java/hivemall/model/HQ.java new file mode 100644 index 00000000..e2516550 --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/HQ.java @@ -0,0 +1,90 @@ +package hivemall.model; + +import com.google.common.io.Resources; +import com.klarna.hiverunner.CommandShellEmulation; +import com.klarna.hiverunner.sql.StatementsSplitter; +import hivemall.model.lazy.LazyMatchingResource; +import hivemall.utils.lang.Preconditions; + +import java.io.File; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; + +abstract public class HQ { + public static List fromStatements(String hq) { + String formatted = CommandShellEmulation.HIVE_CLI.transformScript(hq); + List split = StatementsSplitter.splitStatements(formatted); + List results = new ArrayList(); + for (String q : split) { + results.add(new RawHQ(q)); + } + return results; + } + + public static RawHQ fromStatement(String hq) { + String formatted = CommandShellEmulation.HIVE_CLI.transformScript(hq); + List split = StatementsSplitter.splitStatements(formatted); + + Preconditions.checkArgument( + 1 == split.size(), + "Detected %d queries, should be exactly one. Use `HQ.fromStatements` for multi queries.", + split.size()); + + return new RawHQ(split.get(0)); + } + + public static LazyMatchingResource autoMatchingByFileName(String fileName, Charset charset) { + return new LazyMatchingResource(fileName, charset); + } + + public static LazyMatchingResource autoMatchingByFileName(String fileName) { + return autoMatchingByFileName(fileName, Charset.defaultCharset()); + } + + public static List fromResourcePath(String resourcePath, Charset charset) { + return autoMatchingByFileName(resourcePath, charset).toStrict(""); + } + + public static List fromResourcePath(String resourcePath) { + return fromResourcePath(resourcePath, Charset.defaultCharset()); + } + + public static TableListHQ tableList() { + return new TableListHQ(); + } + + public static CreateTableHQ createTable(String tableName, LinkedHashMap header) { + return new CreateTableHQ(tableName, header); + } + + public static DropTableHQ dropTable(String tableName) { + return new DropTableHQ(tableName); + } + + public static InsertHQ insert(String tableName, List header, List data) { + return new InsertHQ(tableName, header, data); + } + + public static UploadFileToExistingHQ uploadByFullPathToExisting(String tableName, + String fullPath) { + return new UploadFileToExistingHQ(tableName, new File(fullPath)); + } + + public static UploadFileToExistingHQ uploadByResourcePathToExisting(String tableName, + String resourcePath) { + return uploadByFullPathToExisting(tableName, Resources.getResource(resourcePath).getPath()); + } + + public static UploadFileAsNewTableHQ uploadByFullPathAsNewTable(String tableName, + String fullPath, LinkedHashMap header) { + return new UploadFileAsNewTableHQ(tableName, new File(fullPath), header); + } + + public static UploadFileAsNewTableHQ uploadByResourcePathAsNewTable(String tableName, + String resourcePath, LinkedHashMap header) { + return uploadByFullPathAsNewTable(tableName, Resources.getResource(resourcePath).getPath(), + header); + } +} diff --git a/systemtest/src/main/java/hivemall/model/InsertHQ.java b/systemtest/src/main/java/hivemall/model/InsertHQ.java new file mode 100644 index 00000000..0e08a127 --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/InsertHQ.java @@ -0,0 +1,52 @@ +package hivemall.model; + +import hivemall.utils.lang.Preconditions; + +import java.util.List; + +public class InsertHQ extends TableHQ { + public List data; + public List header; + + + InsertHQ(String tableName, List header, List data) { + super(tableName); + + int l = 0; + for (Object[] objs : data) { + Preconditions.checkArgument(objs.length == header.size(), + "l.%d: Mismatch between number of elements in row(%d) and length of header(%d)", l, + objs.length, header.size()); + l++; + } + + this.data = data; + this.header = header; + } + + + public String getAsValuesFormat() { + StringBuilder sb = new StringBuilder(); + for (Object[] row : data) { + sb.append("("); + for (Object val : row) { + sb.append(serialize(val)); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append("),"); + } + sb.deleteCharAt(sb.length() - 1); + + return sb.toString(); + } + + public static String serialize(Object val) { + // TODO array, map + if (val instanceof String) { + return "'" + String.valueOf(val) + "'"; + } else { + return String.valueOf(val); + } + } +} diff --git a/systemtest/src/main/java/hivemall/model/RawHQ.java b/systemtest/src/main/java/hivemall/model/RawHQ.java new file mode 100644 index 00000000..6f695376 --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/RawHQ.java @@ -0,0 +1,15 @@ +package hivemall.model; + +public class RawHQ extends StrictHQ { + String hq; + + + RawHQ(String hq) { + this.hq = hq; + } + + + public String get() { + return hq; + } +} diff --git a/systemtest/src/main/java/hivemall/model/StrictHQ.java b/systemtest/src/main/java/hivemall/model/StrictHQ.java new file mode 100644 index 00000000..6fda38ed --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/StrictHQ.java @@ -0,0 +1,5 @@ +package hivemall.model; + + +abstract public class StrictHQ extends HQ { +} diff --git a/systemtest/src/main/java/hivemall/model/TableHQ.java b/systemtest/src/main/java/hivemall/model/TableHQ.java new file mode 100644 index 00000000..69a7591b --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/TableHQ.java @@ -0,0 +1,10 @@ +package hivemall.model; + +abstract public class TableHQ extends StrictHQ { + public final String tableName; + + + TableHQ(String tableName) { + this.tableName = tableName; + } +} diff --git a/systemtest/src/main/java/hivemall/model/TableListHQ.java b/systemtest/src/main/java/hivemall/model/TableListHQ.java new file mode 100644 index 00000000..f078598f --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/TableListHQ.java @@ -0,0 +1,5 @@ +package hivemall.model; + +public class TableListHQ extends StrictHQ { + TableListHQ() {} +} diff --git a/systemtest/src/main/java/hivemall/model/UploadFileAsNewTableHQ.java b/systemtest/src/main/java/hivemall/model/UploadFileAsNewTableHQ.java new file mode 100644 index 00000000..e53bbad3 --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/UploadFileAsNewTableHQ.java @@ -0,0 +1,19 @@ +package hivemall.model; + +import hivemall.utils.lang.Preconditions; + +import java.io.File; +import java.util.LinkedHashMap; + +public class UploadFileAsNewTableHQ extends UploadFileHQ { + public final LinkedHashMap header; + + + UploadFileAsNewTableHQ(String tableName, File file, LinkedHashMap header) { + super(tableName, file); + + Preconditions.checkArgument(file.exists(), "%s is not found", file.getPath()); + + this.header = header; + } +} diff --git a/systemtest/src/main/java/hivemall/model/UploadFileHQ.java b/systemtest/src/main/java/hivemall/model/UploadFileHQ.java new file mode 100644 index 00000000..d0770dbe --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/UploadFileHQ.java @@ -0,0 +1,38 @@ +package hivemall.model; + +import hivemall.utils.lang.Preconditions; + +import java.io.File; + +public abstract class UploadFileHQ extends TableHQ { + public enum Format { + MSGPACK, CSV, TSV, UNKNOWN + } + + public final File file; + public final Format format; + + + UploadFileHQ(String tableName, File file) { + super(tableName); + + Preconditions.checkArgument(file.exists(), "%s is not found", file.getPath()); + + this.file = file; + this.format = guessFormat(file); + } + + + Format guessFormat(File file) { + String fileName = file.getName(); + if (fileName.endsWith(".msgpack.gz")) { + return Format.MSGPACK; + } else if (fileName.endsWith(".csv")) { + return Format.CSV; + } else if (fileName.endsWith(".tsv")) { + return Format.TSV; + } else { + return Format.UNKNOWN; + } + } +} diff --git a/systemtest/src/main/java/hivemall/model/UploadFileToExistingHQ.java b/systemtest/src/main/java/hivemall/model/UploadFileToExistingHQ.java new file mode 100644 index 00000000..3860cd96 --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/UploadFileToExistingHQ.java @@ -0,0 +1,9 @@ +package hivemall.model; + +import java.io.File; + +public class UploadFileToExistingHQ extends UploadFileHQ { + UploadFileToExistingHQ(String tableName, File file) { + super(tableName, file); + } +} diff --git a/systemtest/src/main/java/hivemall/model/lazy/LazyMatchingResource.java b/systemtest/src/main/java/hivemall/model/lazy/LazyMatchingResource.java new file mode 100644 index 00000000..0df4271c --- /dev/null +++ b/systemtest/src/main/java/hivemall/model/lazy/LazyMatchingResource.java @@ -0,0 +1,38 @@ +package hivemall.model.lazy; + +import com.klarna.hiverunner.CommandShellEmulation; +import com.klarna.hiverunner.sql.StatementsSplitter; +import hivemall.model.HQ; +import hivemall.model.RawHQ; +import hivemall.utils.IO; + +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; + +public class LazyMatchingResource { + private String fileName; + private Charset charset; + + + public LazyMatchingResource(String fileName, Charset charset) { + this.fileName = fileName; + this.charset = charset; + } + + + public List toStrict(String caseDir) { + String query = IO.getFromResourcePath(caseDir + fileName, charset); + String formatted = CommandShellEmulation.HIVE_CLI.transformScript(query); + List split = StatementsSplitter.splitStatements(formatted); + List results = new ArrayList(); + for (String q : split) { + results.add(HQ.fromStatement(q)); + } + return results; + } + + public String[] getAnswers(String answerDir) { + return IO.getFromResourcePath(answerDir + fileName).split(IO.QD); + } +} diff --git a/systemtest/src/main/java/hivemall/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/runner/HiveSystemTestRunner.java new file mode 100644 index 00000000..fe1155d5 --- /dev/null +++ b/systemtest/src/main/java/hivemall/runner/HiveSystemTestRunner.java @@ -0,0 +1,104 @@ +package hivemall.runner; + +import com.klarna.hiverunner.Extractor; +import com.klarna.hiverunner.HiveServerContainer; +import com.klarna.hiverunner.HiveServerContext; +import com.klarna.hiverunner.HiveShell; +import com.klarna.hiverunner.builder.HiveShellBuilder; +import com.klarna.hiverunner.config.HiveRunnerConfig; +import hivemall.model.RawHQ; +import hivemall.model.UploadFileToExistingHQ; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +public class HiveSystemTestRunner extends SystemTestRunner { + private HiveServerContainer container; + private TemporaryFolder tmpFolder; + private HiveShell hShell; + + + public HiveSystemTestRunner(SystemTestCommonInfo ci) { + super(ci); + } + + + @Override + protected void initRunner() { + try { + tmpFolder = new TemporaryFolder() { + { + create(); + getRoot().setWritable(true, false); + } + }; + final HiveRunnerConfig config = new HiveRunnerConfig() { + { + setHiveExecutionEngine("mr"); + } + }; + HiveServerContext ctx = Extractor.getStandaloneHiveServerContext(tmpFolder, config); + container = Extractor.getHiveServerContainer(ctx); + HiveShellBuilder builder = new HiveShellBuilder() { + { + putAllProperties(new HashMap() { + { + put("LOCAL.HDFS.DIR", "${hadoop.tmp.dir}"); + } + }); + setCommandShellEmulation(config.getCommandShellEmulation()); + setHiveServerContainer(container); + } + }; + + hShell = builder.buildShell(); + hShell.start(); + } catch (IOException ex) { + throw new RuntimeException("Failed to init HiveRunner. " + ex.getMessage()); + } + } + + @Override + protected void finRunner() { + if (container != null) + container.tearDown(); + if (tmpFolder != null) + tmpFolder.delete(); + } + + @Override + protected List exec(RawHQ hq) { + logger.info("executing: `" + hq.get() + "`"); + + return hShell.executeQuery(hq.get()); + } + + @Override + List uploadFileToExisting(final UploadFileToExistingHQ hq) throws Exception { + logger.info("executing: insert " + hq.file.getPath() + " into " + hq.tableName + " on " + + dbName); + + switch (hq.format) { + case CSV: + hShell.insertInto(dbName, hq.tableName) + .addRowsFromDelimited(hq.file, ",", null) + .commit(); + break; + case TSV: + hShell.insertInto(dbName, hq.tableName).addRowsFromTsv(hq.file).commit(); + break; + case MSGPACK: + case UNKNOWN: + throw new Exception("Input csv or tsv"); + } + + return new ArrayList() { + { + add("uploaded " + hq.file.getName() + " into " + hq.tableName); + } + }; + } +} diff --git a/systemtest/src/main/java/hivemall/runner/SystemTestCommonInfo.java b/systemtest/src/main/java/hivemall/runner/SystemTestCommonInfo.java new file mode 100644 index 00000000..56c1dbd7 --- /dev/null +++ b/systemtest/src/main/java/hivemall/runner/SystemTestCommonInfo.java @@ -0,0 +1,18 @@ +package hivemall.runner; + +public class SystemTestCommonInfo { + public final String baseDir; + public final String caseDir; + public final String answerDir; + public final String initDir; + public final String dbName; + + + public SystemTestCommonInfo(Class clazz) { + baseDir = clazz.getName().replace(".", "/"); + caseDir = baseDir + "/case/"; + answerDir = baseDir + "/answer/"; + initDir = baseDir + "/init/"; + dbName = clazz.getName().replace(".", "_"); + } +} diff --git a/systemtest/src/main/java/hivemall/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/runner/SystemTestRunner.java new file mode 100644 index 00000000..f3005b0b --- /dev/null +++ b/systemtest/src/main/java/hivemall/runner/SystemTestRunner.java @@ -0,0 +1,194 @@ +package hivemall.runner; + +import hivemall.model.CreateTableHQ; +import hivemall.model.DropTableHQ; +import hivemall.model.HQ; +import hivemall.model.InsertHQ; +import hivemall.model.RawHQ; +import hivemall.model.StrictHQ; +import hivemall.model.TableHQ; +import hivemall.model.TableListHQ; +import hivemall.model.UploadFileAsNewTableHQ; +import hivemall.model.UploadFileHQ; +import hivemall.model.UploadFileToExistingHQ; +import hivemall.utils.IO; +import org.junit.Assert; +import org.junit.rules.ExternalResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public abstract class SystemTestRunner extends ExternalResource { + static final Logger logger = LoggerFactory.getLogger(SystemTestRunner.class); + List classInitHqs = new ArrayList(); + Set immutableTables = new HashSet(); + final String dbName; + + + SystemTestRunner(SystemTestCommonInfo ci) { + this.dbName = formatDBName(ci.dbName); + } + + + @Override + protected void before() throws Exception { + initRunner(); + prepareDB(); // initialize database + } + + @Override + protected void after() { + try { + resetDB(); // clean up database + } catch (Exception ex) { + throw new RuntimeException("Failed to clean up temporary database. " + ex.getMessage()); + } finally { + finRunner(); + } + } + + abstract void initRunner(); + + abstract void finRunner(); + + public void initBy(StrictHQ hq) { + classInitHqs.add(hq); + } + + public void initBy(List hqs) { + classInitHqs.addAll(hqs); + } + + // fix to temporary database and user-defined init (should be called per Test class) + final void prepareDB() throws Exception { + createDB(dbName); + use(dbName); + for (StrictHQ q : classInitHqs) { + exec(q); + + if (q instanceof CreateTableHQ) { + // memo table initialized each class as immutable + immutableTables.add(((CreateTableHQ) q).tableName); + } else if (q instanceof UploadFileAsNewTableHQ) { + immutableTables.add(((UploadFileAsNewTableHQ) q).tableName); + } + } + } + + // drop temporary database (should be called per Test class) + final void resetDB() throws Exception { + dropDB(dbName); + } + + String formatDBName(String dbName) { + return dbName; + } + + public final boolean isImmutableTable(String tableName) { + return immutableTables.contains(tableName); + } + + // execute StrictHQ + public List exec(StrictHQ hq) throws Exception { + if (hq instanceof RawHQ) + return exec((RawHQ) hq); + else if (hq instanceof TableHQ) + return exec((TableHQ) hq); + else if (hq instanceof TableListHQ) + return tableList(); + else + throw new RuntimeException("Unexpected query type: " + hq.getClass()); + } + + //// execute RawHQ + abstract protected List exec(RawHQ hq) throws Exception; + + //// execute TableHQ + protected List exec(TableHQ hq) throws Exception { + if (hq instanceof CreateTableHQ) + return createTable((CreateTableHQ) hq); + else if (hq instanceof DropTableHQ) + return dropTable((DropTableHQ) hq); + else if (hq instanceof InsertHQ) + return insert((InsertHQ) hq); + else if (hq instanceof UploadFileHQ) + return exec((UploadFileHQ) hq); + else + throw new RuntimeException("Unexpected query type: " + hq.getClass()); + } + + ////// execute UploadFileHQ + protected List exec(UploadFileHQ hq) throws Exception { + if (hq instanceof UploadFileAsNewTableHQ) + return uploadFileAsNewTable((UploadFileAsNewTableHQ) hq); + else if (hq instanceof UploadFileToExistingHQ) + return uploadFileToExisting((UploadFileToExistingHQ) hq); + else + throw new RuntimeException("Unexpected query type: " + hq.getClass()); + } + + // matching StrictHQ + public void matching(StrictHQ hq, String answer) throws Exception { + List result = exec(hq); + + Assert.assertArrayEquals(answer.split(IO.RD), result.toArray()); + } + + List createDB(String dbName) throws Exception { + logger.info("executing: create database if not exists" + dbName); + + return exec(HQ.fromStatement("CREATE DATABASE IF NOT EXISTS " + dbName)); + } + + List dropDB(String dbName) throws Exception { + logger.info("executing: drop database if exists " + dbName); + + return exec(HQ.fromStatement("DROP DATABASE IF EXISTS " + dbName + " CASCADE")); + } + + List use(String dbName) throws Exception { + logger.info("executing: use " + dbName); + + return exec(HQ.fromStatement("USE " + dbName)); + } + + List tableList() throws Exception { + logger.info("executing: show tables on " + dbName); + + return exec(HQ.fromStatement("SHOW TABLES")); + } + + List createTable(CreateTableHQ hq) throws Exception { + logger.info("executing: create table " + hq.tableName + " if not exists on " + dbName); + + return exec(HQ.fromStatement("CREATE TABLE IF NOT EXISTS " + hq.tableName + + hq.getTableDeclaration())); + } + + List dropTable(DropTableHQ hq) throws Exception { + logger.info("executing: drop table " + hq.tableName + " if exists on " + dbName); + + return exec(HQ.fromStatement("DROP TABLE IF EXISTS " + hq.tableName)); + } + + List insert(InsertHQ hq) throws Exception { + logger.info("executing: insert into " + hq.tableName + " on " + dbName); + + return exec(HQ.fromStatement("INSERT INTO TABLE " + hq.tableName + " VALUES " + + hq.getAsValuesFormat())); + } + + List uploadFileAsNewTable(UploadFileAsNewTableHQ hq) throws Exception { + logger.info("executing: create " + hq.tableName + " based on " + hq.file.getPath() + + " if not exists on " + dbName); + + createTable(HQ.createTable(hq.tableName, hq.header)); + return uploadFileToExisting(HQ.uploadByFullPathToExisting(hq.tableName, hq.file.getPath())); + } + + abstract List uploadFileToExisting(UploadFileToExistingHQ hq) throws Exception; +} diff --git a/systemtest/src/main/java/hivemall/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/runner/SystemTestTeam.java new file mode 100644 index 00000000..4e02fdc2 --- /dev/null +++ b/systemtest/src/main/java/hivemall/runner/SystemTestTeam.java @@ -0,0 +1,107 @@ +package hivemall.runner; + +import hivemall.model.HQ; +import hivemall.model.RawHQ; +import hivemall.model.StrictHQ; +import hivemall.model.lazy.LazyMatchingResource; +import hivemall.utils.lang.Preconditions; +import org.junit.rules.ExternalResource; + +import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; + +public class SystemTestTeam extends ExternalResource { + private List runners = new ArrayList(); + private List reachGoal = new ArrayList(); // distinct + + private List initHqs = new ArrayList(); + private Set> entries = new LinkedHashSet>(); + + + public SystemTestTeam(SystemTestRunner... runners) { + this.runners.addAll(Arrays.asList(runners)); + } + + + @Override + protected void after() { + for (SystemTestRunner runner : reachGoal) { + try { + List tables = runner.exec(HQ.tableList()); + for (String t : tables) { + if (!runner.isImmutableTable(t)) { + runner.exec(HQ.dropTable(t)); + } + } + } catch (Exception ex) { + throw new RuntimeException("Failed to resetPerMethod database. " + ex.getMessage()); + } + } + } + + // add additional runner for each @Test method + public void add(SystemTestRunner... runners) { + this.runners.addAll(Arrays.asList(runners)); + } + + // add initialization for each @Test method + public void initBy(StrictHQ hq) { + initHqs.add(hq); + } + + public void initBy(List hqs) { + initHqs.addAll(hqs); + } + + public void set(StrictHQ hq, String expected) { + entries.add(pair(hq, expected)); + } + + public void set(List hqs, String... expecteds) { + Preconditions.checkArgument(hqs.size() == expecteds.length, + "Mismatch between number of queries(%d) and length of answers(%d)", hqs.size(), + expecteds.length); + + for (int i = 0; i < expecteds.length; i++) { + set(hqs.get(i), expecteds[i]); + } + } + + public void set(LazyMatchingResource hq, SystemTestCommonInfo ci) { + List rhqs = hq.toStrict(ci.caseDir); + String[] answers = hq.getAnswers(ci.answerDir); + + Preconditions.checkArgument(rhqs.size() == answers.length, + "Mismatch between number of queries(%d) and length of answers(%d)", rhqs.size(), + answers.length); + + for (int i = 0; i < answers.length; i++) { + set(rhqs.get(i), answers[i]); + } + } + + public void run() throws Exception { + for (SystemTestRunner runner : runners) { + if (!reachGoal.contains(runner)) { + // initialization each @Test methods + for (StrictHQ q : initHqs) { + runner.exec(q); + } + reachGoal.add(runner); + } + + for (Entry entry : entries) { + runner.matching(entry.getKey(), entry.getValue()); + } + } + } + + private Entry pair(StrictHQ hq, String answer) { + return new SimpleEntry(hq, answer); + } +} diff --git a/systemtest/src/main/java/hivemall/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/runner/TDSystemTestRunner.java new file mode 100644 index 00000000..977dad89 --- /dev/null +++ b/systemtest/src/main/java/hivemall/runner/TDSystemTestRunner.java @@ -0,0 +1,341 @@ +package hivemall.runner; + +import com.google.common.base.Function; +import com.treasuredata.client.ExponentialBackOff; +import com.treasuredata.client.TDClient; +import com.treasuredata.client.model.TDBulkImportSession; +import com.treasuredata.client.model.TDColumn; +import com.treasuredata.client.model.TDColumnType; +import com.treasuredata.client.model.TDJobRequest; +import com.treasuredata.client.model.TDJobSummary; +import com.treasuredata.client.model.TDResultFormat; +import com.treasuredata.client.model.TDTable; +import hivemall.ConvertToMsgpack; +import hivemall.model.CreateTableHQ; +import hivemall.model.DropTableHQ; +import hivemall.model.HQ; +import hivemall.model.InsertHQ; +import hivemall.model.RawHQ; +import hivemall.model.UploadFileAsNewTableHQ; +import hivemall.model.UploadFileToExistingHQ; +import hivemall.utils.IO; +import org.apache.commons.csv.CSVFormat; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class TDSystemTestRunner extends SystemTestRunner { + private TDClient client; + + + public TDSystemTestRunner(SystemTestCommonInfo ci) { + super(ci); + } + + + @Override + String formatDBName(String name) { + return name.toLowerCase(); + } + + @Override + protected void initRunner() { + client = TDClient.newClient(); + } + + @Override + protected void finRunner() { + if (client != null) + client.close(); + } + + @Override + protected List exec(RawHQ hq) throws Exception { + logger.info("executing: `" + hq.get() + "`"); + + TDJobRequest req = TDJobRequest.newHiveQuery(dbName, hq.get()); + String id = client.submit(req); + + ExponentialBackOff backOff = new ExponentialBackOff(); + TDJobSummary job = client.jobStatus(id); + while (!job.getStatus().isFinished()) { + Thread.sleep(backOff.nextWaitTimeMillis()); + job = client.jobStatus(id); + } + + return client.jobResult(id, TDResultFormat.TSV, new Function>() { + @Override + public List apply(InputStream input) { + List results = new ArrayList(); + BufferedReader reader = null; + try { + try { + reader = new BufferedReader(new InputStreamReader(input)); + String line; + while ((line = reader.readLine()) != null) { + results.addAll(Arrays.asList(line.split(IO.RD))); + } + } finally { + if (reader != null) + reader.close(); + } + } catch (IOException ex) { + throw new RuntimeException("Failed to read results from TD. " + ex.getMessage()); + } + return results; + } + }); + } + + @Override + protected List createDB(final String dbName) throws Exception { + logger.info("executing: create database if not exists " + dbName); + + client.createDatabaseIfNotExists(dbName); + return new ArrayList() { + { + add("created " + dbName); + } + }; + } + + @Override + protected List dropDB(final String dbName) throws Exception { + logger.info("executing: drop database if exists " + dbName); + + client.deleteDatabaseIfExists(dbName); + return new ArrayList() { + { + add("dropped " + dbName); + } + }; + } + + @Override + protected List use(final String dbName) throws Exception { + return new ArrayList() { + { + add("No need to execute `USE` statement on TD, so skipped `USE " + dbName + "`"); + } + }; + } + + @Override + protected List tableList() throws Exception { + logger.info("executing: show tables on " + dbName); + + List tables = client.listTables(dbName); + List result = new ArrayList(); + for (TDTable t : tables) { + result.add(t.getName()); + } + return result; + } + + @Override + protected List createTable(final CreateTableHQ hq) throws Exception { + logger.info("executing: create table " + hq.tableName + " if not exists on " + dbName); + + List columns = new ArrayList(); + for (Map.Entry e : hq.header.entrySet()) { + columns.add(new TDColumn(e.getKey(), TDColumnType.parseColumnType(e.getValue()))); + } + client.createTableIfNotExists(dbName, hq.tableName); + client.updateTableSchema(dbName, hq.tableName, columns); + return new ArrayList() { + { + add("created " + hq.tableName + " on " + dbName); + } + }; + } + + @Override + protected List dropTable(final DropTableHQ hq) throws Exception { + logger.info("executing: drop table " + hq.tableName + " if exists on " + dbName); + + client.deleteTableIfExists(dbName, hq.tableName); + return new ArrayList() { + { + add("dropped " + hq.tableName); + } + }; + } + + @Override + // for Hive 0.13.0 or lower + // if on 0.14.0 or later, doesn't need following + protected List insert(InsertHQ hq) throws Exception { + logger.info("executing: insert into " + hq.tableName + " on " + dbName); + + // construct statement based on WITH clause + StringBuilder sb = new StringBuilder(); + sb.append("WITH t AS ("); + for (Object[] row : hq.data) { + sb.append("SELECT "); + for (int i = 0; i < hq.header.size(); i++) { + sb.append(InsertHQ.serialize(row[i])); + sb.append(" "); + sb.append(hq.header.get(i)); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(" UNION ALL "); + } + sb.delete(sb.length() - 11, sb.length()); + sb.append(") INSERT INTO TABLE "); + sb.append(hq.tableName); + sb.append(" SELECT * FROM t"); + + return exec(HQ.fromStatement(sb.toString())); + } + + @Override + protected List uploadFileAsNewTable(final UploadFileAsNewTableHQ hq) throws Exception { + logger.info("executing: create " + hq.tableName + " based on " + hq.file.getPath() + + " if not exists on " + dbName); + + createTable(HQ.createTable(hq.tableName, hq.header)); + + String sessionName = "session-" + String.valueOf(System.currentTimeMillis()); + String partName = "part-of-" + String.valueOf(sessionName); + client.createBulkImportSession(sessionName, dbName, hq.tableName); + + try { + // upload file as msgpack + switch (hq.format) { + case MSGPACK: + client.uploadBulkImportPart(sessionName, partName, hq.file); + break; + case CSV: { + File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + to.deleteOnExit(); + + client.uploadBulkImportPart(sessionName, partName, + new ConvertToMsgpack(hq.file, new ArrayList(hq.header.keySet()), + CSVFormat.DEFAULT).asFile(to)); + break; + } + case TSV: { + File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + to.deleteOnExit(); + + client.uploadBulkImportPart(sessionName, partName, + new ConvertToMsgpack(hq.file, new ArrayList(hq.header.keySet()), + CSVFormat.TDF).asFile(to)); + break; + } + case UNKNOWN: + throw new Exception("Input msgpack.gz, csv or tsv"); + } + + client.freezeBulkImportSession(sessionName); + client.performBulkImportSession(sessionName); + ExponentialBackOff backOff = new ExponentialBackOff(); + TDBulkImportSession session = client.getBulkImportSession(sessionName); + while (session.getStatus() == TDBulkImportSession.ImportStatus.PERFORMING) { + logger.debug("Waiting bulk import completion"); + Thread.sleep(backOff.nextWaitTimeMillis()); + session = client.getBulkImportSession(sessionName); + } + + client.commitBulkImportSession(sessionName); + session = client.getBulkImportSession(sessionName); + while (session.getStatus() != TDBulkImportSession.ImportStatus.COMMITTED) { + logger.info("Waiting bulk import perform step completion"); + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); + session = client.getBulkImportSession(sessionName); + } + } finally { + client.deleteBulkImportSession(sessionName); + } + + return new ArrayList() { + { + add("uploaded " + hq.file.getName() + " into " + hq.tableName); + } + }; + } + + @Override + protected List uploadFileToExisting(final UploadFileToExistingHQ hq) throws Exception { + logger.info("executing: insert " + hq.file.getPath() + " into " + hq.tableName + " on " + + dbName); + + String sessionName = "session-" + String.valueOf(System.currentTimeMillis()); + String partName = "part-of-" + String.valueOf(sessionName); + client.createBulkImportSession(sessionName, dbName, hq.tableName); + + try { + // upload file as msgpack + switch (hq.format) { + case MSGPACK: + client.uploadBulkImportPart(sessionName, partName, hq.file); + break; + case CSV: { + File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + to.deleteOnExit(); + client.uploadBulkImportPart(sessionName, partName, new ConvertToMsgpack( + hq.file, getHeaderFromTD(hq.tableName), CSVFormat.DEFAULT).asFile(to)); + break; + } + case TSV: { + File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + to.deleteOnExit(); + client.uploadBulkImportPart(sessionName, partName, new ConvertToMsgpack( + hq.file, getHeaderFromTD(hq.tableName), CSVFormat.TDF).asFile(to)); + break; + } + case UNKNOWN: + throw new Exception("Input msgpack.gz, csv or tsv"); + } + + client.freezeBulkImportSession(sessionName); + client.performBulkImportSession(sessionName); + ExponentialBackOff backOff = new ExponentialBackOff(); + TDBulkImportSession session = client.getBulkImportSession(sessionName); + while (session.getStatus() == TDBulkImportSession.ImportStatus.PERFORMING) { + logger.debug("Waiting bulk import completion"); + Thread.sleep(backOff.nextWaitTimeMillis()); + session = client.getBulkImportSession(sessionName); + } + + client.commitBulkImportSession(sessionName); + session = client.getBulkImportSession(sessionName); + while (session.getStatus() != TDBulkImportSession.ImportStatus.COMMITTED) { + logger.info("Waiting bulk import perform step completion"); + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); + session = client.getBulkImportSession(sessionName); + } + } finally { + client.deleteBulkImportSession(sessionName); + } + + return new ArrayList() { + { + add("uploaded " + hq.file.getName() + " into " + hq.tableName); + } + }; + } + + private List getHeaderFromTD(String tableName) { + List header = new ArrayList(); + for (TDTable t : client.listTables(dbName)) { + if (t.getName().equals(tableName)) { + List cols = t.getColumns(); + for (TDColumn col : cols) { + header.add(col.getName()); + } + break; + } + } + return header; + } +} diff --git a/systemtest/src/main/java/hivemall/utils/IO.java b/systemtest/src/main/java/hivemall/utils/IO.java new file mode 100644 index 00000000..7028f876 --- /dev/null +++ b/systemtest/src/main/java/hivemall/utils/IO.java @@ -0,0 +1,60 @@ +package hivemall.utils; + +import com.google.common.io.Resources; +import hivemall.utils.lang.Preconditions; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; + +public class IO { + public static final String RD = "\t"; // row delimiter + public static final String QD = "\n"; // query delimiter + + + private IO() {} + + + public static String getFromFullPath(String fullPath, Charset charset) { + Preconditions.checkArgument(new File(fullPath).exists(), "%s is not found", fullPath); + + return new String(readAllBytes(fullPath), charset); + } + + public static String getFromFullPath(String fullPath) { + return getFromFullPath(fullPath, Charset.defaultCharset()); + } + + public static String getFromResourcePath(String resourcePath, Charset charset) { + String fullPath = Resources.getResource(resourcePath).getPath(); + return getFromFullPath(fullPath, charset); + } + + public static String getFromResourcePath(String resourcePath) { + return getFromResourcePath(resourcePath, Charset.defaultCharset()); + } + + private static byte[] readAllBytes(String filePath) { + File f = new File(filePath); + + int len = (int) f.length(); + byte[] buf = new byte[len]; + + InputStream is = null; + try { + try { + is = new FileInputStream(f); + is.read(buf); + } finally { + if (is != null) + is.close(); + } + } catch (IOException ex) { + throw new RuntimeException("Failed to read " + filePath + ". " + ex.getMessage()); + } + + return buf; + } +} From 4d2bdfc6dfc675d7e693a80e8b32899fc1ba281a Mon Sep 17 00:00:00 2001 From: amaya Date: Mon, 5 Sep 2016 15:04:12 +0900 Subject: [PATCH 02/28] add usage --- systemtest/README.md | 103 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 systemtest/README.md diff --git a/systemtest/README.md b/systemtest/README.md new file mode 100644 index 00000000..363f52ef --- /dev/null +++ b/systemtest/README.md @@ -0,0 +1,103 @@ +## Usage + +### Initialization + +Define `CommonInfo`, `Runner` and `Team` in each your test class. + +### `CommonInfo` + +* `SystemTestCommonInfo` + +`CommonInfo` holds common information of test class, for example, +you can refer to auto-defined path to resources. This should be defined as `private static`. + + +### `Runner` + +* `HiveSystemTestRunner` +* `TDSystemTestRunner` + +`Runner` represents a test environment and its configuration. This must be defined with `@ClassRule` +as `public static` because of JUnit spec. You can add test class initializations by `#initBy(...)` +with class methods of `HQ`, which are abstract domain-specific hive queries, in instance initializer +of each `Runner`. + + +### `Team` + +* `SystemTestTeam` + +`Team` manages `Runner`s each test method. This must be defined with `@Rule` as `public` because of +JUnit spec. You can set `Runner`s via constructor argument as common in class and via `#add(...)` +as method-local and add test method initializations by `#initBy(...)` and test case by `#set(...)` +with class methods of `HQ`. Finally, don't forget call `#run()` to enable set `HQ`s. +As an alternative method, by `#set(HQ.autoMatchingByFileName(${filename}))` with queries predefined in +`auto-defined/path/init/${filename}`, `auto-defined/path/case/${filename}` and +`auto-defined/path/answer/${filename}`, you can do auto matching test. + + +## Notice + +* DDL and insert statement should be called via class methods of `HQ` because of wrapping hive queries +and several runner-specific APIs, don't call them via string statement +* Also you can use low-level API via an instance of `Runner`, independent of `Team` +* You can use `IO.getFromResourcePath(...)` to get answer whose format is TSV + + +## Quick example + +```java +package hivemall; +// here is several imports +public class HogeTest { + private static SystemTestCommonInfo ci = new SystemTestCommonInfo(HogeTest.class); + + @ClassRule + public static HiveSystemTestRunner hRunner = new HiveSystemTestRunner(ci) { + { + initBy(HQ.uploadByResourcePathAsNewTable("people", ci.initDir + "people.tsv", + new LinkedHashMap() { + { + put("name", "string"); + put("age", "int"); + put("sex", "string"); + } + })); + initBy(HQ.fromResourcePath(ci.initDir + "init")); + } + }; + + @Rule + public SystemTestTeam team = new SystemTestTeam(hRunner); + + + @Test + public void insertAndSelectTest() throws Exception { + String tableName = "people0"; + team.initBy(HQ.createTable(tableName, new LinkedHashMap() { + { + put("name", "string"); + put("age", "int"); + put("sex", "string"); + } + })); + team.initBy(HQ.insert(tableName, Arrays.asList("name", "age", "sex"), Arrays.asList( + new Object[]{"Noah", 46, "Male"}, new Object[]{"Isabella", 20, "Female"}))); + team.set(HQ.fromStatement("SELECT name FROM people WHERE age = 20"), "Jacob\tIsabella"); + team.set(HQ.fromStatement("SELECT COUNT(1) FROM " + tableName), "2"); + team.run(); + } +} +``` + +The above needs `systemtest/src/test/resources/hivemall/HogeTest/init/people.tsv` (`systemtest/src/test/resources/${path/to/package}/${classname}/init/people.tsv`) + +```tsv +Jacob 20 Male +Mason 22 Male +Sophia 35 Female +Ethan 55 Male +Emma 15 Female +Noah 46 Male +Isabella 20 Female +``` From 6e31bbd7f7e1df05735d26529936b7309686f599 Mon Sep 17 00:00:00 2001 From: amaya Date: Mon, 5 Sep 2016 18:36:32 +0900 Subject: [PATCH 03/28] move systemtest into specific package --- .../{ => systemtest}/ConvertToMsgpack.java | 2 +- .../{ => systemtest}/model/CreateTableHQ.java | 2 +- .../{ => systemtest}/model/DropTableHQ.java | 2 +- .../hivemall/{ => systemtest}/model/HQ.java | 4 +-- .../{ => systemtest}/model/InsertHQ.java | 2 +- .../{ => systemtest}/model/RawHQ.java | 2 +- .../{ => systemtest}/model/StrictHQ.java | 2 +- .../{ => systemtest}/model/TableHQ.java | 2 +- .../{ => systemtest}/model/TableListHQ.java | 2 +- .../model/UploadFileAsNewTableHQ.java | 2 +- .../{ => systemtest}/model/UploadFileHQ.java | 2 +- .../model/UploadFileToExistingHQ.java | 2 +- .../model/lazy/LazyMatchingResource.java | 8 +++--- .../runner/HiveSystemTestRunner.java | 6 ++-- .../runner/SystemTestCommonInfo.java | 2 +- .../runner/SystemTestRunner.java | 28 +++++++++---------- .../runner/SystemTestTeam.java | 10 +++---- .../runner/TDSystemTestRunner.java | 20 ++++++------- .../hivemall/{ => systemtest}/utils/IO.java | 2 +- 19 files changed, 51 insertions(+), 51 deletions(-) rename systemtest/src/main/java/hivemall/{ => systemtest}/ConvertToMsgpack.java (98%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/CreateTableHQ.java (95%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/DropTableHQ.java (76%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/HQ.java (97%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/InsertHQ.java (97%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/RawHQ.java (82%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/StrictHQ.java (57%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/TableHQ.java (82%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/TableListHQ.java (66%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/UploadFileAsNewTableHQ.java (93%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/UploadFileHQ.java (96%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/UploadFileToExistingHQ.java (83%) rename systemtest/src/main/java/hivemall/{ => systemtest}/model/lazy/LazyMatchingResource.java (86%) rename systemtest/src/main/java/hivemall/{ => systemtest}/runner/HiveSystemTestRunner.java (95%) rename systemtest/src/main/java/hivemall/{ => systemtest}/runner/SystemTestCommonInfo.java (93%) rename systemtest/src/main/java/hivemall/{ => systemtest}/runner/SystemTestRunner.java (91%) rename systemtest/src/main/java/hivemall/{ => systemtest}/runner/SystemTestTeam.java (93%) rename systemtest/src/main/java/hivemall/{ => systemtest}/runner/TDSystemTestRunner.java (96%) rename systemtest/src/main/java/hivemall/{ => systemtest}/utils/IO.java (97%) diff --git a/systemtest/src/main/java/hivemall/ConvertToMsgpack.java b/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java similarity index 98% rename from systemtest/src/main/java/hivemall/ConvertToMsgpack.java rename to systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java index adebc3ec..c9604c0c 100644 --- a/systemtest/src/main/java/hivemall/ConvertToMsgpack.java +++ b/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java @@ -1,4 +1,4 @@ -package hivemall; +package hivemall.systemtest; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; diff --git a/systemtest/src/main/java/hivemall/model/CreateTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java similarity index 95% rename from systemtest/src/main/java/hivemall/model/CreateTableHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java index 7e79f2e8..16a8dd45 100644 --- a/systemtest/src/main/java/hivemall/model/CreateTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; import java.util.LinkedHashMap; import java.util.Map; diff --git a/systemtest/src/main/java/hivemall/model/DropTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java similarity index 76% rename from systemtest/src/main/java/hivemall/model/DropTableHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java index 31dd2d84..6914a6d6 100644 --- a/systemtest/src/main/java/hivemall/model/DropTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; public class DropTableHQ extends TableHQ { DropTableHQ(String tableName) { diff --git a/systemtest/src/main/java/hivemall/model/HQ.java b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java similarity index 97% rename from systemtest/src/main/java/hivemall/model/HQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/HQ.java index e2516550..cb9d5ac9 100644 --- a/systemtest/src/main/java/hivemall/model/HQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java @@ -1,9 +1,9 @@ -package hivemall.model; +package hivemall.systemtest.model; import com.google.common.io.Resources; import com.klarna.hiverunner.CommandShellEmulation; import com.klarna.hiverunner.sql.StatementsSplitter; -import hivemall.model.lazy.LazyMatchingResource; +import hivemall.systemtest.model.lazy.LazyMatchingResource; import hivemall.utils.lang.Preconditions; import java.io.File; diff --git a/systemtest/src/main/java/hivemall/model/InsertHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java similarity index 97% rename from systemtest/src/main/java/hivemall/model/InsertHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java index 0e08a127..db743637 100644 --- a/systemtest/src/main/java/hivemall/model/InsertHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; import hivemall.utils.lang.Preconditions; diff --git a/systemtest/src/main/java/hivemall/model/RawHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java similarity index 82% rename from systemtest/src/main/java/hivemall/model/RawHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java index 6f695376..7967c298 100644 --- a/systemtest/src/main/java/hivemall/model/RawHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; public class RawHQ extends StrictHQ { String hq; diff --git a/systemtest/src/main/java/hivemall/model/StrictHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java similarity index 57% rename from systemtest/src/main/java/hivemall/model/StrictHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java index 6fda38ed..dfb1dadb 100644 --- a/systemtest/src/main/java/hivemall/model/StrictHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; abstract public class StrictHQ extends HQ { diff --git a/systemtest/src/main/java/hivemall/model/TableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java similarity index 82% rename from systemtest/src/main/java/hivemall/model/TableHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java index 69a7591b..9d221daa 100644 --- a/systemtest/src/main/java/hivemall/model/TableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; abstract public class TableHQ extends StrictHQ { public final String tableName; diff --git a/systemtest/src/main/java/hivemall/model/TableListHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java similarity index 66% rename from systemtest/src/main/java/hivemall/model/TableListHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java index f078598f..7c8af92f 100644 --- a/systemtest/src/main/java/hivemall/model/TableListHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; public class TableListHQ extends StrictHQ { TableListHQ() {} diff --git a/systemtest/src/main/java/hivemall/model/UploadFileAsNewTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java similarity index 93% rename from systemtest/src/main/java/hivemall/model/UploadFileAsNewTableHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java index e53bbad3..d3cf5b2a 100644 --- a/systemtest/src/main/java/hivemall/model/UploadFileAsNewTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; import hivemall.utils.lang.Preconditions; diff --git a/systemtest/src/main/java/hivemall/model/UploadFileHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java similarity index 96% rename from systemtest/src/main/java/hivemall/model/UploadFileHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java index d0770dbe..e88370c4 100644 --- a/systemtest/src/main/java/hivemall/model/UploadFileHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; import hivemall.utils.lang.Preconditions; diff --git a/systemtest/src/main/java/hivemall/model/UploadFileToExistingHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java similarity index 83% rename from systemtest/src/main/java/hivemall/model/UploadFileToExistingHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java index 3860cd96..1244aabc 100644 --- a/systemtest/src/main/java/hivemall/model/UploadFileToExistingHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java @@ -1,4 +1,4 @@ -package hivemall.model; +package hivemall.systemtest.model; import java.io.File; diff --git a/systemtest/src/main/java/hivemall/model/lazy/LazyMatchingResource.java b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java similarity index 86% rename from systemtest/src/main/java/hivemall/model/lazy/LazyMatchingResource.java rename to systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java index 0df4271c..70c7f52c 100644 --- a/systemtest/src/main/java/hivemall/model/lazy/LazyMatchingResource.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java @@ -1,10 +1,10 @@ -package hivemall.model.lazy; +package hivemall.systemtest.model.lazy; import com.klarna.hiverunner.CommandShellEmulation; import com.klarna.hiverunner.sql.StatementsSplitter; -import hivemall.model.HQ; -import hivemall.model.RawHQ; -import hivemall.utils.IO; +import hivemall.systemtest.model.HQ; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.utils.IO; import java.nio.charset.Charset; import java.util.ArrayList; diff --git a/systemtest/src/main/java/hivemall/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java similarity index 95% rename from systemtest/src/main/java/hivemall/runner/HiveSystemTestRunner.java rename to systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java index fe1155d5..461735f8 100644 --- a/systemtest/src/main/java/hivemall/runner/HiveSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java @@ -1,4 +1,4 @@ -package hivemall.runner; +package hivemall.systemtest.runner; import com.klarna.hiverunner.Extractor; import com.klarna.hiverunner.HiveServerContainer; @@ -6,8 +6,8 @@ import com.klarna.hiverunner.HiveShell; import com.klarna.hiverunner.builder.HiveShellBuilder; import com.klarna.hiverunner.config.HiveRunnerConfig; -import hivemall.model.RawHQ; -import hivemall.model.UploadFileToExistingHQ; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.model.UploadFileToExistingHQ; import org.junit.rules.TemporaryFolder; import java.io.IOException; diff --git a/systemtest/src/main/java/hivemall/runner/SystemTestCommonInfo.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java similarity index 93% rename from systemtest/src/main/java/hivemall/runner/SystemTestCommonInfo.java rename to systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java index 56c1dbd7..992bc683 100644 --- a/systemtest/src/main/java/hivemall/runner/SystemTestCommonInfo.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java @@ -1,4 +1,4 @@ -package hivemall.runner; +package hivemall.systemtest.runner; public class SystemTestCommonInfo { public final String baseDir; diff --git a/systemtest/src/main/java/hivemall/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java similarity index 91% rename from systemtest/src/main/java/hivemall/runner/SystemTestRunner.java rename to systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index f3005b0b..4808751f 100644 --- a/systemtest/src/main/java/hivemall/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -1,17 +1,17 @@ -package hivemall.runner; - -import hivemall.model.CreateTableHQ; -import hivemall.model.DropTableHQ; -import hivemall.model.HQ; -import hivemall.model.InsertHQ; -import hivemall.model.RawHQ; -import hivemall.model.StrictHQ; -import hivemall.model.TableHQ; -import hivemall.model.TableListHQ; -import hivemall.model.UploadFileAsNewTableHQ; -import hivemall.model.UploadFileHQ; -import hivemall.model.UploadFileToExistingHQ; -import hivemall.utils.IO; +package hivemall.systemtest.runner; + +import hivemall.systemtest.model.CreateTableHQ; +import hivemall.systemtest.model.DropTableHQ; +import hivemall.systemtest.model.HQ; +import hivemall.systemtest.model.InsertHQ; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.model.StrictHQ; +import hivemall.systemtest.model.TableHQ; +import hivemall.systemtest.model.TableListHQ; +import hivemall.systemtest.model.UploadFileAsNewTableHQ; +import hivemall.systemtest.model.UploadFileHQ; +import hivemall.systemtest.model.UploadFileToExistingHQ; +import hivemall.systemtest.utils.IO; import org.junit.Assert; import org.junit.rules.ExternalResource; import org.slf4j.Logger; diff --git a/systemtest/src/main/java/hivemall/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java similarity index 93% rename from systemtest/src/main/java/hivemall/runner/SystemTestTeam.java rename to systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index 4e02fdc2..abc8e3ec 100644 --- a/systemtest/src/main/java/hivemall/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -1,9 +1,9 @@ -package hivemall.runner; +package hivemall.systemtest.runner; -import hivemall.model.HQ; -import hivemall.model.RawHQ; -import hivemall.model.StrictHQ; -import hivemall.model.lazy.LazyMatchingResource; +import hivemall.systemtest.model.HQ; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.model.StrictHQ; +import hivemall.systemtest.model.lazy.LazyMatchingResource; import hivemall.utils.lang.Preconditions; import org.junit.rules.ExternalResource; diff --git a/systemtest/src/main/java/hivemall/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java similarity index 96% rename from systemtest/src/main/java/hivemall/runner/TDSystemTestRunner.java rename to systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index 977dad89..4f97daa4 100644 --- a/systemtest/src/main/java/hivemall/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -1,4 +1,4 @@ -package hivemall.runner; +package hivemall.systemtest.runner; import com.google.common.base.Function; import com.treasuredata.client.ExponentialBackOff; @@ -10,15 +10,15 @@ import com.treasuredata.client.model.TDJobSummary; import com.treasuredata.client.model.TDResultFormat; import com.treasuredata.client.model.TDTable; -import hivemall.ConvertToMsgpack; -import hivemall.model.CreateTableHQ; -import hivemall.model.DropTableHQ; -import hivemall.model.HQ; -import hivemall.model.InsertHQ; -import hivemall.model.RawHQ; -import hivemall.model.UploadFileAsNewTableHQ; -import hivemall.model.UploadFileToExistingHQ; -import hivemall.utils.IO; +import hivemall.systemtest.ConvertToMsgpack; +import hivemall.systemtest.model.CreateTableHQ; +import hivemall.systemtest.model.DropTableHQ; +import hivemall.systemtest.model.HQ; +import hivemall.systemtest.model.InsertHQ; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.model.UploadFileAsNewTableHQ; +import hivemall.systemtest.model.UploadFileToExistingHQ; +import hivemall.systemtest.utils.IO; import org.apache.commons.csv.CSVFormat; import java.io.BufferedReader; diff --git a/systemtest/src/main/java/hivemall/utils/IO.java b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java similarity index 97% rename from systemtest/src/main/java/hivemall/utils/IO.java rename to systemtest/src/main/java/hivemall/systemtest/utils/IO.java index 7028f876..2cb93369 100644 --- a/systemtest/src/main/java/hivemall/utils/IO.java +++ b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java @@ -1,4 +1,4 @@ -package hivemall.utils; +package hivemall.systemtest.utils; import com.google.common.io.Resources; import hivemall.utils.lang.Preconditions; From d0108a90d12a4900cb690ce2e8c00c3385187853 Mon Sep 17 00:00:00 2001 From: amaya Date: Mon, 5 Sep 2016 18:37:00 +0900 Subject: [PATCH 04/28] mod project settings of systemtest --- systemtest/pom.xml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/systemtest/pom.xml b/systemtest/pom.xml index d58c64a7..3c4f9c88 100644 --- a/systemtest/pom.xml +++ b/systemtest/pom.xml @@ -2,14 +2,18 @@ + 4.0.0 + - hivemall io.github.myui + hivemall 0.4.2-rc.2 + ../pom.xml - 4.0.0 - systemtest + hivemall-systemtest + System test for Hivemall + jar From 1a0da21ed3986fa036f3e4b756ad75d8b89d7d90 Mon Sep 17 00:00:00 2001 From: amaya Date: Mon, 5 Sep 2016 18:46:18 +0900 Subject: [PATCH 05/28] add ordered/unordered result matching --- systemtest/pom.xml | 5 +++++ .../systemtest/runner/SystemTestRunner.java | 19 +++++++++++++++++-- .../systemtest/runner/SystemTestTeam.java | 17 +++++++++++------ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/systemtest/pom.xml b/systemtest/pom.xml index 3c4f9c88..9485f59c 100644 --- a/systemtest/pom.xml +++ b/systemtest/pom.xml @@ -47,5 +47,10 @@ msgpack-core 0.8.9 + + org.hamcrest + hamcrest-library + 1.3 + diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index 4808751f..c37bfcec 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -12,12 +12,14 @@ import hivemall.systemtest.model.UploadFileHQ; import hivemall.systemtest.model.UploadFileToExistingHQ; import hivemall.systemtest.utils.IO; +import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.rules.ExternalResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -132,10 +134,23 @@ else if (hq instanceof UploadFileToExistingHQ) } // matching StrictHQ - public void matching(StrictHQ hq, String answer) throws Exception { + public void matching(StrictHQ hq, String answer, boolean ordered) throws Exception { List result = exec(hq); - Assert.assertArrayEquals(answer.split(IO.RD), result.toArray()); + if (ordered) { + // take order into consideration (like list) + Assert.assertThat(Arrays.asList(answer.split(IO.RD)), + Matchers.contains(result.toArray())); + } else { + // not take order into consideration (like multiset) + Assert.assertThat(Arrays.asList(answer.split(IO.RD)), + Matchers.containsInAnyOrder(result.toArray())); + } + } + + // matching StrictHQ (ordered == false) + public void matching(StrictHQ hq, String answer) throws Exception { + matching(hq, answer, false); } List createDB(String dbName) throws Exception { diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index abc8e3ec..9c2b39e2 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -10,17 +10,17 @@ import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Arrays; -import java.util.LinkedHashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Map.Entry; -import java.util.Set; public class SystemTestTeam extends ExternalResource { private List runners = new ArrayList(); private List reachGoal = new ArrayList(); // distinct private List initHqs = new ArrayList(); - private Set> entries = new LinkedHashSet>(); + private Map, Boolean> entries = new LinkedHashMap, Boolean>(); public SystemTestTeam(SystemTestRunner... runners) { @@ -58,8 +58,12 @@ public void initBy(List hqs) { initHqs.addAll(hqs); } + public void set(StrictHQ hq, String expected, boolean ordered) { + entries.put(pair(hq, expected), ordered); + } + public void set(StrictHQ hq, String expected) { - entries.add(pair(hq, expected)); + entries.put(pair(hq, expected), false); } public void set(List hqs, String... expecteds) { @@ -95,8 +99,9 @@ public void run() throws Exception { reachGoal.add(runner); } - for (Entry entry : entries) { - runner.matching(entry.getKey(), entry.getValue()); + for (Entry, Boolean> entry : entries.entrySet()) { + runner.matching(entry.getKey().getKey(), entry.getKey().getValue(), + entry.getValue()); } } } From 81efb8c0bf457ddec271cf02259cd244555e62f7 Mon Sep 17 00:00:00 2001 From: amaya Date: Mon, 5 Sep 2016 19:08:36 +0900 Subject: [PATCH 06/28] add license header --- systemtest/pom.xml | 32 +++++++++++++++++++ .../java/com/klarna/hiverunner/Extractor.java | 18 +++++++++++ .../hivemall/systemtest/ConvertToMsgpack.java | 18 +++++++++++ .../systemtest/model/CreateTableHQ.java | 18 +++++++++++ .../systemtest/model/DropTableHQ.java | 18 +++++++++++ .../java/hivemall/systemtest/model/HQ.java | 18 +++++++++++ .../hivemall/systemtest/model/InsertHQ.java | 18 +++++++++++ .../java/hivemall/systemtest/model/RawHQ.java | 18 +++++++++++ .../hivemall/systemtest/model/StrictHQ.java | 18 +++++++++++ .../hivemall/systemtest/model/TableHQ.java | 18 +++++++++++ .../systemtest/model/TableListHQ.java | 18 +++++++++++ .../model/UploadFileAsNewTableHQ.java | 18 +++++++++++ .../systemtest/model/UploadFileHQ.java | 18 +++++++++++ .../model/UploadFileToExistingHQ.java | 18 +++++++++++ .../model/lazy/LazyMatchingResource.java | 18 +++++++++++ .../runner/HiveSystemTestRunner.java | 18 +++++++++++ .../runner/SystemTestCommonInfo.java | 18 +++++++++++ .../systemtest/runner/SystemTestRunner.java | 18 +++++++++++ .../systemtest/runner/SystemTestTeam.java | 18 +++++++++++ .../systemtest/runner/TDSystemTestRunner.java | 18 +++++++++++ .../java/hivemall/systemtest/utils/IO.java | 18 +++++++++++ 21 files changed, 392 insertions(+) diff --git a/systemtest/pom.xml b/systemtest/pom.xml index 9485f59c..e59d2ce6 100644 --- a/systemtest/pom.xml +++ b/systemtest/pom.xml @@ -53,4 +53,36 @@ 1.3 + + + target + target/classes + ${project.artifactId}-${project.version} + target/test-classes + + + + com.mycila + license-maven-plugin + 2.8 + +
${project.parent.basedir}/resources/license-header.txt
+ + ${build.year} + ${project.organization.name} + + + src/main/**/*.java + src/test/**/*.java + + UTF-8 + + ${project.parent.basedir}/resources/header-definition.xml + + + + + + + diff --git a/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java b/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java index 5ab39089..99720f02 100644 --- a/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java +++ b/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.klarna.hiverunner; import com.klarna.hiverunner.config.HiveRunnerConfig; diff --git a/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java b/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java index c9604c0c..eb2ff692 100644 --- a/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java +++ b/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest; import org.apache.commons.csv.CSVFormat; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java index 16a8dd45..5366e1b4 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; import java.util.LinkedHashMap; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java index 6914a6d6..2740d2f0 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; public class DropTableHQ extends TableHQ { diff --git a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java index cb9d5ac9..81b23cbf 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; import com.google.common.io.Resources; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java index db743637..71a3895a 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; import hivemall.utils.lang.Preconditions; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java index 7967c298..6827b3f5 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; public class RawHQ extends StrictHQ { diff --git a/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java index dfb1dadb..0b5d7dc5 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java index 9d221daa..ebf0f879 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; abstract public class TableHQ extends StrictHQ { diff --git a/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java index 7c8af92f..4b47cde8 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; public class TableListHQ extends StrictHQ { diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java index d3cf5b2a..a2ac5a38 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; import hivemall.utils.lang.Preconditions; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java index e88370c4..209864eb 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; import hivemall.utils.lang.Preconditions; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java index 1244aabc..16fe3b9c 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model; import java.io.File; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java index 70c7f52c..0d961ec3 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.model.lazy; import com.klarna.hiverunner.CommandShellEmulation; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java index 461735f8..efb27b58 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.runner; import com.klarna.hiverunner.Extractor; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java index 992bc683..05709424 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.runner; public class SystemTestCommonInfo { diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index c37bfcec..b3bafead 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.runner; import hivemall.systemtest.model.CreateTableHQ; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index 9c2b39e2..145ea25d 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.runner; import hivemall.systemtest.model.HQ; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index 4f97daa4..de25ec12 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.runner; import com.google.common.base.Function; diff --git a/systemtest/src/main/java/hivemall/systemtest/utils/IO.java b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java index 2cb93369..f0767b71 100644 --- a/systemtest/src/main/java/hivemall/systemtest/utils/IO.java +++ b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java @@ -1,3 +1,21 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package hivemall.systemtest.utils; import com.google.common.io.Resources; From 6bc276c9fdf2a0ec3bea057e7500bc68947e980a Mon Sep 17 00:00:00 2001 From: amaya Date: Wed, 7 Sep 2016 14:01:31 +0900 Subject: [PATCH 07/28] enable to use external properties --- .../runner/HiveSystemTestRunner.java | 20 +++++- .../systemtest/runner/SystemTestRunner.java | 23 ++++++- .../systemtest/runner/TDSystemTestRunner.java | 61 +++++++++++++++++-- .../resources/hivemall/hiverunner.properties | 6 ++ .../src/test/resources/hivemall/td.properties | 13 ++++ 5 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 systemtest/src/test/resources/hivemall/hiverunner.properties create mode 100644 systemtest/src/test/resources/hivemall/td.properties diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java index efb27b58..2ab0c5b9 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java @@ -18,6 +18,7 @@ */ package hivemall.systemtest.runner; +import com.klarna.hiverunner.CommandShellEmulation; import com.klarna.hiverunner.Extractor; import com.klarna.hiverunner.HiveServerContainer; import com.klarna.hiverunner.HiveServerContext; @@ -39,8 +40,12 @@ public class HiveSystemTestRunner extends SystemTestRunner { private HiveShell hShell; + public HiveSystemTestRunner(SystemTestCommonInfo ci, String propertiesFile) { + super(ci, propertiesFile); + } + public HiveSystemTestRunner(SystemTestCommonInfo ci) { - super(ci); + super(ci, "hiverunner.properties"); } @@ -55,7 +60,18 @@ protected void initRunner() { }; final HiveRunnerConfig config = new HiveRunnerConfig() { { - setHiveExecutionEngine("mr"); + // required + setHiveExecutionEngine(props.getProperty("hive.execution.engine", "mr")); + + // optional + if (props.containsKey("enableTimeout")) + setTimeoutEnabled(Boolean.valueOf(props.getProperty("enableTimeout"))); + if (props.containsKey("timeoutRetryLimit")) + setTimeoutRetries(Integer.valueOf(props.getProperty("timeoutRetryLimit"))); + if (props.containsKey("timeoutSeconds")) + setTimeoutSeconds(Integer.valueOf(props.getProperty("timeoutSeconds"))); + if (props.containsKey("commandShellEmulation")) + setCommandShellEmulation(CommandShellEmulation.valueOf(props.getProperty("commandShellEmulation"))); } }; HiveServerContext ctx = Extractor.getStandaloneHiveServerContext(tmpFolder, config); diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index b3bafead..fbb20a96 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -18,6 +18,7 @@ */ package hivemall.systemtest.runner; +import com.google.common.io.Resources; import hivemall.systemtest.model.CreateTableHQ; import hivemall.systemtest.model.DropTableHQ; import hivemall.systemtest.model.HQ; @@ -36,10 +37,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.FileInputStream; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Properties; import java.util.Set; public abstract class SystemTestRunner extends ExternalResource { @@ -47,10 +51,27 @@ public abstract class SystemTestRunner extends ExternalResource { List classInitHqs = new ArrayList(); Set immutableTables = new HashSet(); final String dbName; + final Properties props; - SystemTestRunner(SystemTestCommonInfo ci) { + SystemTestRunner(SystemTestCommonInfo ci, String propertiesFile) { this.dbName = formatDBName(ci.dbName); + + final String path = "hivemall/" + propertiesFile; + try { + InputStream is = null; + try { + props = new Properties(); + is = new FileInputStream(Resources.getResource(path).getPath()); + props.load(is); + } finally { + if (is != null) + is.close(); + } + } catch (Exception ex) { + throw new RuntimeException("Failed to load properties from " + path + ". " + + ex.getMessage()); + } } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index de25ec12..8946ea48 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -48,14 +48,24 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.TimeUnit; public class TDSystemTestRunner extends SystemTestRunner { private TDClient client; + private int execFinishRetryLimit = 7; + private int fileUploadPerformRetryLimit = 7; + private int fileUploadCommitBackOff = 5; + private int fileUploadCommitRetryLimit = 7; + + + public TDSystemTestRunner(SystemTestCommonInfo ci, String propertiesFile) { + super(ci, propertiesFile); + } public TDSystemTestRunner(SystemTestCommonInfo ci) { - super(ci); + super(ci, "td.properties"); } @@ -66,7 +76,26 @@ String formatDBName(String name) { @Override protected void initRunner() { - client = TDClient.newClient(); + // optional + if (props.containsKey("execFinishRetryLimit")) + execFinishRetryLimit = Integer.valueOf(props.getProperty("execFinishRetryLimit")); + if (props.containsKey("fileUploadPerformRetryLimit")) + fileUploadPerformRetryLimit = Integer.valueOf(props.getProperty("fileUploadPerformRetryLimit")); + if (props.containsKey("fileUploadCommitBackOff")) + fileUploadCommitBackOff = Integer.valueOf(props.getProperty("fileUploadCommitBackOff")); + if (props.containsKey("fileUploadCommitRetryLimit")) + fileUploadCommitRetryLimit = Integer.valueOf(props.getProperty("fileUploadCommitRetryLimit")); + + Properties TDPorps = System.getProperties(); + for (Map.Entry e : props.entrySet()) { + if (e.getKey().toString().startsWith("td.client.")) { + TDPorps.setProperty(e.getKey().toString(), e.getValue().toString()); + } + } + System.setProperties(TDPorps); + + client = System.getProperties().size() == TDPorps.size() ? TDClient.newClient() // use $HOME/.td/td.conf + : TDClient.newBuilder(false).build(); // use *.properties } @Override @@ -84,9 +113,17 @@ protected List exec(RawHQ hq) throws Exception { ExponentialBackOff backOff = new ExponentialBackOff(); TDJobSummary job = client.jobStatus(id); + int nRetries = 0; while (!job.getStatus().isFinished()) { + if (nRetries > execFinishRetryLimit) { + throw new Exception("Exceed standard of finish check retry repetition: " + + execFinishRetryLimit); + } + Thread.sleep(backOff.nextWaitTimeMillis()); job = client.jobStatus(id); + + nRetries++; } return client.jobResult(id, TDResultFormat.TSV, new Function>() { @@ -258,18 +295,34 @@ protected List uploadFileAsNewTable(final UploadFileAsNewTableHQ hq) thr client.performBulkImportSession(sessionName); ExponentialBackOff backOff = new ExponentialBackOff(); TDBulkImportSession session = client.getBulkImportSession(sessionName); + int performNRetries = 0; while (session.getStatus() == TDBulkImportSession.ImportStatus.PERFORMING) { + if (performNRetries > fileUploadPerformRetryLimit) { + throw new Exception("Exceed standard of perform check retry repetition: " + + fileUploadPerformRetryLimit); + } + logger.debug("Waiting bulk import completion"); Thread.sleep(backOff.nextWaitTimeMillis()); session = client.getBulkImportSession(sessionName); + + performNRetries++; } client.commitBulkImportSession(sessionName); session = client.getBulkImportSession(sessionName); + int commitNRetries = 0; while (session.getStatus() != TDBulkImportSession.ImportStatus.COMMITTED) { + if (commitNRetries > fileUploadCommitRetryLimit) { + throw new Exception("Exceed standard of commit check retry repetition: " + + fileUploadCommitRetryLimit); + } + logger.info("Waiting bulk import perform step completion"); - Thread.sleep(TimeUnit.SECONDS.toMillis(5)); + Thread.sleep(TimeUnit.SECONDS.toMillis(fileUploadCommitBackOff)); session = client.getBulkImportSession(sessionName); + + commitNRetries++; } } finally { client.deleteBulkImportSession(sessionName); @@ -329,7 +382,7 @@ protected List uploadFileToExisting(final UploadFileToExistingHQ hq) thr session = client.getBulkImportSession(sessionName); while (session.getStatus() != TDBulkImportSession.ImportStatus.COMMITTED) { logger.info("Waiting bulk import perform step completion"); - Thread.sleep(TimeUnit.SECONDS.toMillis(5)); + Thread.sleep(TimeUnit.SECONDS.toMillis(fileUploadCommitBackOff)); session = client.getBulkImportSession(sessionName); } } finally { diff --git a/systemtest/src/test/resources/hivemall/hiverunner.properties b/systemtest/src/test/resources/hivemall/hiverunner.properties new file mode 100644 index 00000000..5870bf15 --- /dev/null +++ b/systemtest/src/test/resources/hivemall/hiverunner.properties @@ -0,0 +1,6 @@ +# properties for HiveRunner +hive.execution.engine=mr +#enableTimeout=false +#timeoutRetryLimit=2 +#timeoutSeconds=30 +#commandShellEmulation=HIVE_CLI diff --git a/systemtest/src/test/resources/hivemall/td.properties b/systemtest/src/test/resources/hivemall/td.properties new file mode 100644 index 00000000..4ba88f3c --- /dev/null +++ b/systemtest/src/test/resources/hivemall/td.properties @@ -0,0 +1,13 @@ +# properties for TD +execFinishRetryLimit=7 +fileUploadPerformRetryLimit=7 +fileUploadCommitBackOff=5 +fileUploadCommitRetryLimit=7 + +# TD client configs like following prior to $HOME/.td/td.conf +#td.client.user= +#td.client.apikey= +#td.client.password= +#td.client.endpoint= +#td.client.port= +#td.client.usessl= From 915cf7873600cec181a57ed52467da2068326922 Mon Sep 17 00:00:00 2001 From: amaya Date: Wed, 7 Sep 2016 14:15:35 +0900 Subject: [PATCH 08/28] update README about external properties --- systemtest/README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/systemtest/README.md b/systemtest/README.md index 363f52ef..001318e2 100644 --- a/systemtest/README.md +++ b/systemtest/README.md @@ -4,7 +4,7 @@ Define `CommonInfo`, `Runner` and `Team` in each your test class. -### `CommonInfo` +#### `CommonInfo` * `SystemTestCommonInfo` @@ -12,7 +12,7 @@ Define `CommonInfo`, `Runner` and `Team` in each your test class. you can refer to auto-defined path to resources. This should be defined as `private static`. -### `Runner` +#### `Runner` * `HiveSystemTestRunner` * `TDSystemTestRunner` @@ -23,7 +23,7 @@ with class methods of `HQ`, which are abstract domain-specific hive queries, in of each `Runner`. -### `Team` +#### `Team` * `SystemTestTeam` @@ -36,12 +36,20 @@ As an alternative method, by `#set(HQ.autoMatchingByFileName(${filename}))` with `auto-defined/path/answer/${filename}`, you can do auto matching test. +### External properties + +You can use external properties at `systemtest/src/test/resources/hivemall/*`, default is `hiverunner.properties` +for `HiveSystemTestRunner` and `td.properties` for `TDSystemTestRunner`. Also user-defined properties file can +be loaded via constructor of `Runner` by file name. + + ## Notice * DDL and insert statement should be called via class methods of `HQ` because of wrapping hive queries and several runner-specific APIs, don't call them via string statement * Also you can use low-level API via an instance of `Runner`, independent of `Team` * You can use `IO.getFromResourcePath(...)` to get answer whose format is TSV +* TD client configs in properties file prior to $HOME/.td/td.conf ## Quick example From bd0143c5e1bf331ad2887e16db74dd5a79262fda Mon Sep 17 00:00:00 2001 From: amaya Date: Wed, 7 Sep 2016 15:07:38 +0900 Subject: [PATCH 09/28] suppress warnings --- .../runner/HiveSystemTestRunner.java | 9 ++-- .../runner/SystemTestCommonInfo.java | 2 +- .../systemtest/runner/TDSystemTestRunner.java | 44 ++++--------------- 3 files changed, 13 insertions(+), 42 deletions(-) diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java index 2ab0c5b9..931b5a22 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java @@ -30,7 +30,7 @@ import org.junit.rules.TemporaryFolder; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -76,6 +76,7 @@ protected void initRunner() { }; HiveServerContext ctx = Extractor.getStandaloneHiveServerContext(tmpFolder, config); container = Extractor.getHiveServerContainer(ctx); + @SuppressWarnings("serial") HiveShellBuilder builder = new HiveShellBuilder() { { putAllProperties(new HashMap() { @@ -129,10 +130,6 @@ List uploadFileToExisting(final UploadFileToExistingHQ hq) throws Except throw new Exception("Input csv or tsv"); } - return new ArrayList() { - { - add("uploaded " + hq.file.getName() + " into " + hq.tableName); - } - }; + return Collections.singletonList("uploaded " + hq.file.getName() + " into " + hq.tableName); } } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java index 05709424..12229d80 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java @@ -26,7 +26,7 @@ public class SystemTestCommonInfo { public final String dbName; - public SystemTestCommonInfo(Class clazz) { + public SystemTestCommonInfo(Class clazz) { baseDir = clazz.getName().replace(".", "/"); caseDir = baseDir + "/case/"; answerDir = baseDir + "/answer/"; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index 8946ea48..668c8477 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -46,6 +46,7 @@ import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; @@ -155,11 +156,7 @@ protected List createDB(final String dbName) throws Exception { logger.info("executing: create database if not exists " + dbName); client.createDatabaseIfNotExists(dbName); - return new ArrayList() { - { - add("created " + dbName); - } - }; + return Collections.singletonList("created " + dbName); } @Override @@ -167,20 +164,13 @@ protected List dropDB(final String dbName) throws Exception { logger.info("executing: drop database if exists " + dbName); client.deleteDatabaseIfExists(dbName); - return new ArrayList() { - { - add("dropped " + dbName); - } - }; + return Collections.singletonList("dropped " + dbName); } @Override protected List use(final String dbName) throws Exception { - return new ArrayList() { - { - add("No need to execute `USE` statement on TD, so skipped `USE " + dbName + "`"); - } - }; + return Collections.singletonList("No need to execute `USE` statement on TD, so skipped `USE " + + dbName + "`"); } @Override @@ -205,11 +195,7 @@ protected List createTable(final CreateTableHQ hq) throws Exception { } client.createTableIfNotExists(dbName, hq.tableName); client.updateTableSchema(dbName, hq.tableName, columns); - return new ArrayList() { - { - add("created " + hq.tableName + " on " + dbName); - } - }; + return Collections.singletonList("created " + hq.tableName + " on " + dbName); } @Override @@ -217,11 +203,7 @@ protected List dropTable(final DropTableHQ hq) throws Exception { logger.info("executing: drop table " + hq.tableName + " if exists on " + dbName); client.deleteTableIfExists(dbName, hq.tableName); - return new ArrayList() { - { - add("dropped " + hq.tableName); - } - }; + return Collections.singletonList("dropped " + hq.tableName); } @Override @@ -328,11 +310,7 @@ protected List uploadFileAsNewTable(final UploadFileAsNewTableHQ hq) thr client.deleteBulkImportSession(sessionName); } - return new ArrayList() { - { - add("uploaded " + hq.file.getName() + " into " + hq.tableName); - } - }; + return Collections.singletonList("uploaded " + hq.file.getName() + " into " + hq.tableName); } @Override @@ -389,11 +367,7 @@ protected List uploadFileToExisting(final UploadFileToExistingHQ hq) thr client.deleteBulkImportSession(sessionName); } - return new ArrayList() { - { - add("uploaded " + hq.file.getName() + " into " + hq.tableName); - } - }; + return Collections.singletonList("uploaded " + hq.file.getName() + " into " + hq.tableName); } private List getHeaderFromTD(String tableName) { From 57566e4a52281889e64b28f8e2f372884b31bb32 Mon Sep 17 00:00:00 2001 From: amaya Date: Wed, 7 Sep 2016 15:33:25 +0900 Subject: [PATCH 10/28] potential bug to exception --- .../systemtest/runner/SystemTestTeam.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index 145ea25d..dac9c288 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -40,6 +40,8 @@ public class SystemTestTeam extends ExternalResource { private List initHqs = new ArrayList(); private Map, Boolean> entries = new LinkedHashMap, Boolean>(); + private boolean needRun = false; // remind `run()` + public SystemTestTeam(SystemTestRunner... runners) { this.runners.addAll(Arrays.asList(runners)); @@ -48,6 +50,10 @@ public SystemTestTeam(SystemTestRunner... runners) { @Override protected void after() { + if (needRun) { + throw new IllegalStateException("Call `SystemTestTeam#run()`"); + } + for (SystemTestRunner runner : reachGoal) { try { List tables = runner.exec(HQ.tableList()); @@ -70,18 +76,26 @@ public void add(SystemTestRunner... runners) { // add initialization for each @Test method public void initBy(StrictHQ hq) { initHqs.add(hq); + + needRun = true; } public void initBy(List hqs) { initHqs.addAll(hqs); + + needRun = true; } public void set(StrictHQ hq, String expected, boolean ordered) { entries.put(pair(hq, expected), ordered); + + needRun = true; } public void set(StrictHQ hq, String expected) { entries.put(pair(hq, expected), false); + + needRun = true; } public void set(List hqs, String... expecteds) { @@ -92,6 +106,8 @@ public void set(List hqs, String... expecteds) { for (int i = 0; i < expecteds.length; i++) { set(hqs.get(i), expecteds[i]); } + + needRun = true; } public void set(LazyMatchingResource hq, SystemTestCommonInfo ci) { @@ -105,9 +121,13 @@ public void set(LazyMatchingResource hq, SystemTestCommonInfo ci) { for (int i = 0; i < answers.length; i++) { set(rhqs.get(i), answers[i]); } + + needRun = true; } public void run() throws Exception { + needRun = false; + for (SystemTestRunner runner : runners) { if (!reachGoal.contains(runner)) { // initialization each @Test methods From 02a9385f188e955da9005f8a94db6d3e908c4589 Mon Sep 17 00:00:00 2001 From: amaya Date: Wed, 7 Sep 2016 17:04:38 +0900 Subject: [PATCH 11/28] add ordered/unordered result matching for multiple queries --- .../systemtest/runner/SystemTestTeam.java | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index dac9c288..f00e3bad 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -98,19 +98,30 @@ public void set(StrictHQ hq, String expected) { needRun = true; } - public void set(List hqs, String... expecteds) { - Preconditions.checkArgument(hqs.size() == expecteds.length, + public void set(List hqs, List expecteds, List ordereds) { + Preconditions.checkArgument(hqs.size() == expecteds.size(), "Mismatch between number of queries(%d) and length of answers(%d)", hqs.size(), - expecteds.length); + expecteds.size()); + Preconditions.checkArgument(hqs.size() == ordereds.size(), + "Mismatch between number of queries(%d) and correspond ordered flags(%d)", hqs.size(), + ordereds.size()); - for (int i = 0; i < expecteds.length; i++) { - set(hqs.get(i), expecteds[i]); + for (int i = 0; i < expecteds.size(); i++) { + set(hqs.get(i), expecteds.get(i), ordereds.get(i)); } needRun = true; } - public void set(LazyMatchingResource hq, SystemTestCommonInfo ci) { + public void set(List hqs, List expecteds) { + List ordereds = new ArrayList(); + for (int i = 0; i < hqs.size(); i++) { + ordereds.add(false); + } + set(hqs, expecteds, ordereds); + } + + public void set(LazyMatchingResource hq, SystemTestCommonInfo ci, boolean ordered) { List rhqs = hq.toStrict(ci.caseDir); String[] answers = hq.getAnswers(ci.answerDir); @@ -119,12 +130,16 @@ public void set(LazyMatchingResource hq, SystemTestCommonInfo ci) { answers.length); for (int i = 0; i < answers.length; i++) { - set(rhqs.get(i), answers[i]); + set(rhqs.get(i), answers[i], ordered); } needRun = true; } + public void set(LazyMatchingResource hq, SystemTestCommonInfo ci) { + set(hq, ci, false); + } + public void run() throws Exception { needRun = false; From bca540407221ec300e73d9fd6b476645863ad7fd Mon Sep 17 00:00:00 2001 From: amaya Date: Wed, 7 Sep 2016 17:55:57 +0900 Subject: [PATCH 12/28] refine README --- systemtest/README.md | 60 ++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/systemtest/README.md b/systemtest/README.md index 001318e2..387b4b75 100644 --- a/systemtest/README.md +++ b/systemtest/README.md @@ -49,29 +49,29 @@ be loaded via constructor of `Runner` by file name. and several runner-specific APIs, don't call them via string statement * Also you can use low-level API via an instance of `Runner`, independent of `Team` * You can use `IO.getFromResourcePath(...)` to get answer whose format is TSV +* Table created in initialization of runner should be used as immutable, don't neither insert nor update * TD client configs in properties file prior to $HOME/.td/td.conf - ## Quick example ```java package hivemall; // here is several imports -public class HogeTest { - private static SystemTestCommonInfo ci = new SystemTestCommonInfo(HogeTest.class); +public class QuickExample { + private static SystemTestCommonInfo ci = new SystemTestCommonInfo(QuickExample.class); @ClassRule public static HiveSystemTestRunner hRunner = new HiveSystemTestRunner(ci) { { - initBy(HQ.uploadByResourcePathAsNewTable("people", ci.initDir + "people.tsv", + initBy(HQ.uploadByResourcePathAsNewTable("color", ci.initDir + "color.tsv", new LinkedHashMap() { { put("name", "string"); - put("age", "int"); - put("sex", "string"); + put("red", "int"); + put("green", "int"); + put("blue", "int"); } - })); - initBy(HQ.fromResourcePath(ci.initDir + "init")); + })); // create table `color`, which is marked as immutable, for this test class } }; @@ -80,32 +80,42 @@ public class HogeTest { @Test - public void insertAndSelectTest() throws Exception { - String tableName = "people0"; + public void test0() throws Exception { + team.set(HQ.fromStatement("SELECT name FROM color WHERE blue = 255 ORDER BY name"), "azure\tblue\tmagenta", true); // ordered test + team.run(); // this call is required + } + + @Test + public void test1() throws Exception { + String tableName = "users"; team.initBy(HQ.createTable(tableName, new LinkedHashMap() { { put("name", "string"); put("age", "int"); - put("sex", "string"); + put("favorite_color", "string"); } - })); - team.initBy(HQ.insert(tableName, Arrays.asList("name", "age", "sex"), Arrays.asList( - new Object[]{"Noah", 46, "Male"}, new Object[]{"Isabella", 20, "Female"}))); - team.set(HQ.fromStatement("SELECT name FROM people WHERE age = 20"), "Jacob\tIsabella"); - team.set(HQ.fromStatement("SELECT COUNT(1) FROM " + tableName), "2"); - team.run(); + })); // create local table in this test method `users` for each set runner(only hRunner here) + team.initBy(HQ.insert(tableName, Arrays.asList("name", "age", "favorite_color"), Arrays.asList( + new Object[]{"Karen", 16, "orange"}, new Object[]{"Alice", 17, "pink"}))); // insert into `users` + team.set(HQ.fromStatement("SELECT CONCAT('rgb(', red, ',', green, ',', blue, ')') FROM " + + tableName + " u LEFT JOIN color c on u.favorite_color = c.name"), "rgb(255,165,0)\trgb(255,192,203)"); // unordered test + team.run(); // this call is required } } ``` -The above needs `systemtest/src/test/resources/hivemall/HogeTest/init/people.tsv` (`systemtest/src/test/resources/${path/to/package}/${classname}/init/people.tsv`) +The above needs `systemtest/src/test/resources/hivemall/HogeTest/init/color.tsv` (`systemtest/src/test/resources/${path/to/package}/${classname}/init/color.tsv`) ```tsv -Jacob 20 Male -Mason 22 Male -Sophia 35 Female -Ethan 55 Male -Emma 15 Female -Noah 46 Male -Isabella 20 Female +blue 0 0 255 +lavender 230 230 250 +magenta 255 0 255 +violet 238 130 238 +purple 128 0 128 +azure 240 255 255 +lightseagreen 32 178 170 +orange 255 165 0 +orangered 255 69 0 +red 255 0 0 +pink 255 192 203 ``` From 48a4fc97e6643cfc2d1fcf96dbe788c0fcadc619 Mon Sep 17 00:00:00 2001 From: amaya Date: Thu, 8 Sep 2016 17:43:30 +0900 Subject: [PATCH 13/28] refine README --- systemtest/README.md | 74 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/systemtest/README.md b/systemtest/README.md index 387b4b75..ee1a81d0 100644 --- a/systemtest/README.md +++ b/systemtest/README.md @@ -62,6 +62,24 @@ public class QuickExample { @ClassRule public static HiveSystemTestRunner hRunner = new HiveSystemTestRunner(ci) { + { + initBy(HQ.uploadByResourcePathAsNewTable("color", ci.initDir + "color.tsv", + new LinkedHashMap() { + { + put("name", "string"); + put("red", "int"); + put("green", "int"); + put("blue", "int"); + } + })); // create table `color`, which is marked as immutable, for this test class + + // add function from hivemall class + initBy(HQ.fromStatement("CREATE TEMPORARY FUNCTION hivemall_version as 'hivemall.HivemallVersionUDF'")); + } + }; + + @ClassRule + public static TDSystemTestRunner tRunner = new TDSystemTestRunner(ci) { { initBy(HQ.uploadByResourcePathAsNewTable("color", ci.initDir + "color.tsv", new LinkedHashMap() { @@ -76,17 +94,22 @@ public class QuickExample { }; @Rule - public SystemTestTeam team = new SystemTestTeam(hRunner); + public SystemTestTeam team = new SystemTestTeam(hRunner); // set hRunner as default runner + + @Rule + public ExpectedException predictor = ExpectedException.none(); @Test public void test0() throws Exception { + team.add(tRunner, hRunner); // test on HiveRunner -> TD -> HiveRunner (NOTE: state of DB is retained in each runner) team.set(HQ.fromStatement("SELECT name FROM color WHERE blue = 255 ORDER BY name"), "azure\tblue\tmagenta", true); // ordered test team.run(); // this call is required } @Test public void test1() throws Exception { + // test on HiveRunner once only String tableName = "users"; team.initBy(HQ.createTable(tableName, new LinkedHashMap() { { @@ -101,10 +124,38 @@ public class QuickExample { + tableName + " u LEFT JOIN color c on u.favorite_color = c.name"), "rgb(255,165,0)\trgb(255,192,203)"); // unordered test team.run(); // this call is required } + + @Test + public void test2() throws Exception { + // You can also use runner's raw API directly + for(RawHQ q: HQ.fromStatements("SELECT hivemall_version();SELECT hivemall_version();")) { + System.out.println(hRunner.exec(q).get(0)); + } + // raw API doesn't require `SystemTestTeam#run()` + } + + @Test + public void test3() throws Exception { + // test on HiveRunner once only + // auto matching by files which name is `test3` in `case/` and `answer/` + team.set(HQ.autoMatchingByFileName("test3", ci)); // unordered test + team.run(); // this call is required + } + + @Test + public void test4() throws Exception { + // test on HiveRunner once only + predictor.expect(Throwable.class); // you can use systemtest w/ other rules + team.set(HQ.fromStatement("invalid queryyy")); // this query throws an exception + team.run(); // this call is required + // thrown exception will be caught by `ExpectedException` rule + } } ``` -The above needs `systemtest/src/test/resources/hivemall/HogeTest/init/color.tsv` (`systemtest/src/test/resources/${path/to/package}/${classname}/init/color.tsv`) +The above requires following files + +* `systemtest/src/test/resources/hivemall/HogeTest/init/color.tsv` (`systemtest/src/test/resources/${path/to/package}/${className}/init/${fileName}`) ```tsv blue 0 0 255 @@ -119,3 +170,22 @@ orangered 255 69 0 red 255 0 0 pink 255 192 203 ``` + +* `systemtest/src/test/resources/hivemall/HogeTest/case/test3` (`systemtest/src/test/resources/${path/to/package}/${className}/case/${fileName}`) + +```sql +-- write your hive queries +-- comments like this and multiple queries in one row are allowed +SELECT blue FROM color WHERE name = 'lavender';SELECT green FROM color WHERE name LIKE 'orange%' +SELECT name FROM color WHERE blue = 255 +``` + +* `systemtest/src/test/resources/hivemall/HogeTest/answer/test3` (`systemtest/src/test/resources/${path/to/package}/${className}/answer/${fileName}`) + +tsv format is required + +```tsv +230 +165 69 +azure blue magenta +``` From e7b2c6b6a4c4f86b1e6f1f2214f163e692ca700a Mon Sep 17 00:00:00 2001 From: amaya Date: Fri, 9 Sep 2016 14:46:27 +0900 Subject: [PATCH 14/28] support array and map in insert statement --- systemtest/README.md | 1 + .../hivemall/systemtest/model/InsertHQ.java | 30 +------- .../systemtest/runner/SystemTestRunner.java | 73 ++++++++++++++++++- .../systemtest/runner/TDSystemTestRunner.java | 29 -------- 4 files changed, 74 insertions(+), 59 deletions(-) diff --git a/systemtest/README.md b/systemtest/README.md index ee1a81d0..9d1442aa 100644 --- a/systemtest/README.md +++ b/systemtest/README.md @@ -51,6 +51,7 @@ and several runner-specific APIs, don't call them via string statement * You can use `IO.getFromResourcePath(...)` to get answer whose format is TSV * Table created in initialization of runner should be used as immutable, don't neither insert nor update * TD client configs in properties file prior to $HOME/.td/td.conf +* Don't use insert w/ big data, use file upload instead ## Quick example diff --git a/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java index 71a3895a..69260c6b 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java @@ -33,38 +33,12 @@ public class InsertHQ extends TableHQ { int l = 0; for (Object[] objs : data) { Preconditions.checkArgument(objs.length == header.size(), - "l.%d: Mismatch between number of elements in row(%d) and length of header(%d)", l, - objs.length, header.size()); + "l.%s : Mismatch between number of elements in row(%s) and length of header(%s)", + l, objs.length, header.size()); l++; } this.data = data; this.header = header; } - - - public String getAsValuesFormat() { - StringBuilder sb = new StringBuilder(); - for (Object[] row : data) { - sb.append("("); - for (Object val : row) { - sb.append(serialize(val)); - sb.append(","); - } - sb.deleteCharAt(sb.length() - 1); - sb.append("),"); - } - sb.deleteCharAt(sb.length() - 1); - - return sb.toString(); - } - - public static String serialize(Object val) { - // TODO array, map - if (val instanceof String) { - return "'" + String.valueOf(val) + "'"; - } else { - return String.valueOf(val); - } - } } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index fbb20a96..93bf6688 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; @@ -232,8 +233,32 @@ List dropTable(DropTableHQ hq) throws Exception { List insert(InsertHQ hq) throws Exception { logger.info("executing: insert into " + hq.tableName + " on " + dbName); - return exec(HQ.fromStatement("INSERT INTO TABLE " + hq.tableName + " VALUES " - + hq.getAsValuesFormat())); + // *WORKAROUND* + // `INSERT INTO TABLE ... VALUES ...` + // cannot use array() and map() with `VALUES` on hiverunner(v3.0.0), + // cannot insert anything on TD(v20160901) + // `WITH ... AS (SELECT ...) INSERT INTO TABLE ... SELECT * FROM ...` + // can insert anything on hiverunner(v3.0.0) + // cannot use map on TD(v20160901) + StringBuilder sb = new StringBuilder(); + sb.append("WITH temporary_table_for_with_clause AS ("); + for (Object[] row : hq.data) { + sb.append("SELECT "); + for (int i = 0; i < hq.header.size(); i++) { + sb.append(serialize(row[i])); + sb.append(" "); + sb.append(hq.header.get(i)); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(" UNION ALL "); + } + sb.delete(sb.length() - 11, sb.length()); + sb.append(") INSERT INTO TABLE "); + sb.append(hq.tableName); + sb.append(" SELECT * FROM temporary_table_for_with_clause"); + + return exec(HQ.fromStatement(sb.toString())); } List uploadFileAsNewTable(UploadFileAsNewTableHQ hq) throws Exception { @@ -245,4 +270,48 @@ List uploadFileAsNewTable(UploadFileAsNewTableHQ hq) throws Exception { } abstract List uploadFileToExisting(UploadFileToExistingHQ hq) throws Exception; + + private String serialize(Object val) { + // NOTE: this method is low-performance, don't use w/ big data + if (val instanceof String) { + return "'" + String.valueOf(val) + "'"; + } else if (val instanceof Object[]) { + Object[] objs = (Object[]) val; + StringBuilder sb = new StringBuilder(); + sb.append("array("); + for (Object o : objs) { + sb.append(serialize(o)); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } else if (val instanceof List) { + List list = (List) val; + StringBuilder sb = new StringBuilder(); + sb.append("array("); + for (Object o : list) { + sb.append(serialize(o)); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } else if (val instanceof Map) { + Map map = (Map) val; + StringBuilder sb = new StringBuilder(); + sb.append("map("); + for (Map.Entry e : map.entrySet()) { + sb.append(serialize(e.getKey())); + sb.append(","); + sb.append(serialize(e.getValue())); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } else { + return String.valueOf(val); + } + } } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index 668c8477..6c381a70 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -32,7 +32,6 @@ import hivemall.systemtest.model.CreateTableHQ; import hivemall.systemtest.model.DropTableHQ; import hivemall.systemtest.model.HQ; -import hivemall.systemtest.model.InsertHQ; import hivemall.systemtest.model.RawHQ; import hivemall.systemtest.model.UploadFileAsNewTableHQ; import hivemall.systemtest.model.UploadFileToExistingHQ; @@ -206,34 +205,6 @@ protected List dropTable(final DropTableHQ hq) throws Exception { return Collections.singletonList("dropped " + hq.tableName); } - @Override - // for Hive 0.13.0 or lower - // if on 0.14.0 or later, doesn't need following - protected List insert(InsertHQ hq) throws Exception { - logger.info("executing: insert into " + hq.tableName + " on " + dbName); - - // construct statement based on WITH clause - StringBuilder sb = new StringBuilder(); - sb.append("WITH t AS ("); - for (Object[] row : hq.data) { - sb.append("SELECT "); - for (int i = 0; i < hq.header.size(); i++) { - sb.append(InsertHQ.serialize(row[i])); - sb.append(" "); - sb.append(hq.header.get(i)); - sb.append(","); - } - sb.deleteCharAt(sb.length() - 1); - sb.append(" UNION ALL "); - } - sb.delete(sb.length() - 11, sb.length()); - sb.append(") INSERT INTO TABLE "); - sb.append(hq.tableName); - sb.append(" SELECT * FROM t"); - - return exec(HQ.fromStatement(sb.toString())); - } - @Override protected List uploadFileAsNewTable(final UploadFileAsNewTableHQ hq) throws Exception { logger.info("executing: create " + hq.tableName + " based on " + hq.file.getPath() From 58b89ec6cec2736d769b6499031b9bd64e4de124 Mon Sep 17 00:00:00 2001 From: amaya Date: Fri, 9 Sep 2016 14:49:35 +0900 Subject: [PATCH 15/28] fix error messages --- systemtest/src/main/java/hivemall/systemtest/model/HQ.java | 2 +- .../java/hivemall/systemtest/runner/SystemTestTeam.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java index 81b23cbf..64d587f3 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java @@ -47,7 +47,7 @@ public static RawHQ fromStatement(String hq) { Preconditions.checkArgument( 1 == split.size(), - "Detected %d queries, should be exactly one. Use `HQ.fromStatements` for multi queries.", + "Detected %s queries, should be exactly one. Use `HQ.fromStatements` for multi queries.", split.size()); return new RawHQ(split.get(0)); diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index f00e3bad..54090862 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -100,10 +100,10 @@ public void set(StrictHQ hq, String expected) { public void set(List hqs, List expecteds, List ordereds) { Preconditions.checkArgument(hqs.size() == expecteds.size(), - "Mismatch between number of queries(%d) and length of answers(%d)", hqs.size(), + "Mismatch between number of queries(%s) and length of answers(%s)", hqs.size(), expecteds.size()); Preconditions.checkArgument(hqs.size() == ordereds.size(), - "Mismatch between number of queries(%d) and correspond ordered flags(%d)", hqs.size(), + "Mismatch between number of queries(%s) and correspond ordered flags(%s)", hqs.size(), ordereds.size()); for (int i = 0; i < expecteds.size(); i++) { @@ -126,7 +126,7 @@ public void set(LazyMatchingResource hq, SystemTestCommonInfo ci, boolean ordere String[] answers = hq.getAnswers(ci.answerDir); Preconditions.checkArgument(rhqs.size() == answers.length, - "Mismatch between number of queries(%d) and length of answers(%d)", rhqs.size(), + "Mismatch between number of queries(%s) and length of answers(%s)", rhqs.size(), answers.length); for (int i = 0; i < answers.length; i++) { From 71d89d22d2025ad98540386dbacc9a085d9bd23b Mon Sep 17 00:00:00 2001 From: amaya Date: Mon, 12 Sep 2016 15:12:32 +0900 Subject: [PATCH 16/28] make code tougher --- .../hivemall/systemtest/ConvertToMsgpack.java | 25 ++-- .../exception/QueryExecutionException.java | 27 ++++ .../systemtest/model/CreateTableHQ.java | 12 +- .../systemtest/model/DropTableHQ.java | 4 +- .../java/hivemall/systemtest/model/HQ.java | 103 ++++++++++---- .../hivemall/systemtest/model/InsertHQ.java | 11 +- .../java/hivemall/systemtest/model/RawHQ.java | 15 +-- .../hivemall/systemtest/model/StrictHQ.java | 2 +- .../hivemall/systemtest/model/TableHQ.java | 8 +- .../model/UploadFileAsNewTableHQ.java | 10 +- .../systemtest/model/UploadFileHQ.java | 13 +- .../model/UploadFileToExistingHQ.java | 3 +- .../model/lazy/LazyMatchingResource.java | 27 ++-- .../runner/HiveSystemTestRunner.java | 40 +++--- .../runner/SystemTestCommonInfo.java | 12 +- .../systemtest/runner/SystemTestRunner.java | 126 ++++++++++-------- .../systemtest/runner/SystemTestTeam.java | 60 ++++++--- .../systemtest/runner/TDSystemTestRunner.java | 90 +++++++------ .../java/hivemall/systemtest/utils/IO.java | 33 +++-- 19 files changed, 388 insertions(+), 233 deletions(-) create mode 100644 systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java diff --git a/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java b/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java index eb2ff692..8d3e9400 100644 --- a/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java +++ b/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java @@ -18,6 +18,7 @@ */ package hivemall.systemtest; +import hivemall.utils.lang.Preconditions; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -25,6 +26,7 @@ import org.msgpack.core.MessagePacker; import org.msgpack.value.ValueFactory; +import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; @@ -42,24 +44,28 @@ public class ConvertToMsgpack { @Nonnull private final CSVFormat format; + public ConvertToMsgpack(@CheckForNull File file, @CheckForNull List header, + @CheckForNull CSVFormat format) { + Preconditions.checkNotNull(file, "file"); + Preconditions.checkNotNull(header, "header"); + Preconditions.checkNotNull(format, "format"); + Preconditions.checkArgument(file.exists(), "%s not found", file.getPath()); - public ConvertToMsgpack(File file, List header, CSVFormat format) { this.file = file; this.header = header; this.format = format; } - public byte[] asByteArray(final boolean needTimeColumn) throws Exception { - ByteArrayOutputStream os = new ByteArrayOutputStream(); - MessagePacker packer = MessagePack.newDefaultPacker(new GZIPOutputStream(os)); - BufferedReader br = new BufferedReader(new FileReader(file)); + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + final MessagePacker packer = MessagePack.newDefaultPacker(new GZIPOutputStream(os)); + final BufferedReader br = new BufferedReader(new FileReader(file)); try { // always skip header, use user-defined or existing table's - CSVParser parser = format.withSkipHeaderRecord().parse(br); + final CSVParser parser = format.withSkipHeaderRecord().parse(br); final long time = System.currentTimeMillis() / 1000; for (CSVRecord record : parser.getRecords()) { - ValueFactory.MapBuilder map = ValueFactory.newMapBuilder(); + final ValueFactory.MapBuilder map = ValueFactory.newMapBuilder(); // add `time` column if needed && not exists if (needTimeColumn && !header.contains("time")) { @@ -86,7 +92,10 @@ public byte[] asByteArray() throws Exception { return asByteArray(true); } - public File asFile(File to, final boolean needTimeColumn) throws Exception { + public File asFile(@CheckForNull File to, final boolean needTimeColumn) throws Exception { + Preconditions.checkNotNull(to, "to"); + Preconditions.checkArgument(to.exists(), "%s not found", to.getPath()); + FileOutputStream os = null; try { os = new FileOutputStream(to); diff --git a/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java b/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java new file mode 100644 index 00000000..e17a32ed --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java @@ -0,0 +1,27 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hivemall.systemtest.exception; + +import javax.annotation.Nonnull; + +public class QueryExecutionException extends RuntimeException { + public QueryExecutionException(@Nonnull final String message) { + super(message); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java index 5366e1b4..40004b8b 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java @@ -18,21 +18,23 @@ */ package hivemall.systemtest.model; +import javax.annotation.Nonnull; import java.util.LinkedHashMap; import java.util.Map; public class CreateTableHQ extends TableHQ { - public LinkedHashMap header; + @Nonnull + public final LinkedHashMap header; - - CreateTableHQ(String tableName, LinkedHashMap header) { + CreateTableHQ(@Nonnull final String tableName, + @Nonnull final LinkedHashMap header) { super(tableName); + this.header = header; } - public String getTableDeclaration() { - StringBuilder sb = new StringBuilder(); + final StringBuilder sb = new StringBuilder(); sb.append("("); for (Map.Entry e : header.entrySet()) { sb.append(e.getKey()); diff --git a/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java index 2740d2f0..4e9fe23d 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java @@ -18,8 +18,10 @@ */ package hivemall.systemtest.model; +import javax.annotation.Nonnull; + public class DropTableHQ extends TableHQ { - DropTableHQ(String tableName) { + DropTableHQ(@Nonnull final String tableName) { super(tableName); } } diff --git a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java index 64d587f3..1e45016c 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java @@ -24,6 +24,8 @@ import hivemall.systemtest.model.lazy.LazyMatchingResource; import hivemall.utils.lang.Preconditions; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import java.io.File; import java.nio.charset.Charset; import java.util.ArrayList; @@ -31,19 +33,12 @@ import java.util.List; abstract public class HQ { - public static List fromStatements(String hq) { - String formatted = CommandShellEmulation.HIVE_CLI.transformScript(hq); - List split = StatementsSplitter.splitStatements(formatted); - List results = new ArrayList(); - for (String q : split) { - results.add(new RawHQ(q)); - } - return results; - } + @Nonnull + public static RawHQ fromStatement(String query) { + Preconditions.checkNotNull(query, "query"); - public static RawHQ fromStatement(String hq) { - String formatted = CommandShellEmulation.HIVE_CLI.transformScript(hq); - List split = StatementsSplitter.splitStatements(formatted); + final String formatted = CommandShellEmulation.HIVE_CLI.transformScript(query); + final List split = StatementsSplitter.splitStatements(formatted); Preconditions.checkArgument( 1 == split.size(), @@ -53,55 +48,107 @@ public static RawHQ fromStatement(String hq) { return new RawHQ(split.get(0)); } - public static LazyMatchingResource autoMatchingByFileName(String fileName, Charset charset) { + @Nonnull + public static List fromStatements(@CheckForNull final String queries) { + Preconditions.checkNotNull(queries, "queries"); + + final String formatted = CommandShellEmulation.HIVE_CLI.transformScript(queries); + final List split = StatementsSplitter.splitStatements(formatted); + final List results = new ArrayList(); + for (String q : split) { + results.add(new RawHQ(q)); + } + return results; + } + + @Nonnull + public static LazyMatchingResource autoMatchingByFileName(@CheckForNull final String fileName, + @CheckForNull final Charset charset) { + Preconditions.checkNotNull(fileName, "fileName"); + Preconditions.checkNotNull(charset, "charset"); + return new LazyMatchingResource(fileName, charset); } - public static LazyMatchingResource autoMatchingByFileName(String fileName) { + @Nonnull + public static LazyMatchingResource autoMatchingByFileName(final String fileName) { return autoMatchingByFileName(fileName, Charset.defaultCharset()); } - public static List fromResourcePath(String resourcePath, Charset charset) { + @Nonnull + public static List fromResourcePath(final String resourcePath, final Charset charset) { return autoMatchingByFileName(resourcePath, charset).toStrict(""); } - public static List fromResourcePath(String resourcePath) { + @Nonnull + public static List fromResourcePath(final String resourcePath) { return fromResourcePath(resourcePath, Charset.defaultCharset()); } + @Nonnull public static TableListHQ tableList() { return new TableListHQ(); } - public static CreateTableHQ createTable(String tableName, LinkedHashMap header) { + @Nonnull + public static CreateTableHQ createTable(@CheckForNull final String tableName, + @CheckForNull final LinkedHashMap header) { + Preconditions.checkNotNull(tableName, "tableName"); + Preconditions.checkNotNull(header, "header"); + return new CreateTableHQ(tableName, header); } - public static DropTableHQ dropTable(String tableName) { + @Nonnull + public static DropTableHQ dropTable(@CheckForNull final String tableName) { + Preconditions.checkNotNull(tableName, "tableName"); + return new DropTableHQ(tableName); } - public static InsertHQ insert(String tableName, List header, List data) { + @Nonnull + public static InsertHQ insert(@CheckForNull final String tableName, + @CheckForNull final List header, @CheckForNull final List data) { + Preconditions.checkNotNull(tableName, "tableName"); + Preconditions.checkNotNull(header, "header"); + Preconditions.checkNotNull(data, "data"); + return new InsertHQ(tableName, header, data); } - public static UploadFileToExistingHQ uploadByFullPathToExisting(String tableName, - String fullPath) { + @Nonnull + public static UploadFileToExistingHQ uploadByFullPathToExisting( + @CheckForNull final String tableName, @CheckForNull final String fullPath) { + Preconditions.checkNotNull(tableName, "tableName"); + Preconditions.checkNotNull(fullPath, "fullPath"); + return new UploadFileToExistingHQ(tableName, new File(fullPath)); } - public static UploadFileToExistingHQ uploadByResourcePathToExisting(String tableName, - String resourcePath) { + @Nonnull + public static UploadFileToExistingHQ uploadByResourcePathToExisting(final String tableName, + final String resourcePath) { return uploadByFullPathToExisting(tableName, Resources.getResource(resourcePath).getPath()); } - public static UploadFileAsNewTableHQ uploadByFullPathAsNewTable(String tableName, - String fullPath, LinkedHashMap header) { - return new UploadFileAsNewTableHQ(tableName, new File(fullPath), header); + @Nonnull + public static UploadFileAsNewTableHQ uploadByFullPathAsNewTable( + @CheckForNull final String tableName, @CheckForNull final String fullPath, + @CheckForNull final LinkedHashMap header) { + Preconditions.checkNotNull(tableName, "tableName"); + Preconditions.checkNotNull(fullPath, "fullPath"); + Preconditions.checkNotNull(header, "header"); + + final File file = new File(fullPath); + + Preconditions.checkArgument(file.exists(), "%s not found", file.getPath()); + + return new UploadFileAsNewTableHQ(tableName, file, header); } - public static UploadFileAsNewTableHQ uploadByResourcePathAsNewTable(String tableName, - String resourcePath, LinkedHashMap header) { + @Nonnull + public static UploadFileAsNewTableHQ uploadByResourcePathAsNewTable(final String tableName, + final String resourcePath, final LinkedHashMap header) { return uploadByFullPathAsNewTable(tableName, Resources.getResource(resourcePath).getPath(), header); } diff --git a/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java index 69260c6b..30bf13b2 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java @@ -20,14 +20,17 @@ import hivemall.utils.lang.Preconditions; +import javax.annotation.Nonnull; import java.util.List; public class InsertHQ extends TableHQ { - public List data; - public List header; + @Nonnull + public final List data; + @Nonnull + public final List header; - - InsertHQ(String tableName, List header, List data) { + InsertHQ(@Nonnull final String tableName, @Nonnull final List header, + @Nonnull final List data) { super(tableName); int l = 0; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java index 6827b3f5..5575509b 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java @@ -18,16 +18,13 @@ */ package hivemall.systemtest.model; -public class RawHQ extends StrictHQ { - String hq; - - - RawHQ(String hq) { - this.hq = hq; - } +import javax.annotation.Nonnull; +public class RawHQ extends StrictHQ { + @Nonnull + public final String query; - public String get() { - return hq; + RawHQ(@Nonnull final String query) { + this.query = query; } } diff --git a/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java index 0b5d7dc5..921e1939 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java @@ -19,5 +19,5 @@ package hivemall.systemtest.model; -abstract public class StrictHQ extends HQ { +public abstract class StrictHQ extends HQ { } diff --git a/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java index ebf0f879..e6b9aec3 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java @@ -18,11 +18,13 @@ */ package hivemall.systemtest.model; -abstract public class TableHQ extends StrictHQ { - public final String tableName; +import javax.annotation.Nonnull; +public abstract class TableHQ extends StrictHQ { + @Nonnull + public final String tableName; - TableHQ(String tableName) { + TableHQ(@Nonnull final String tableName) { this.tableName = tableName; } } diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java index a2ac5a38..d88976e3 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java @@ -18,20 +18,18 @@ */ package hivemall.systemtest.model; -import hivemall.utils.lang.Preconditions; - +import javax.annotation.Nonnull; import java.io.File; import java.util.LinkedHashMap; public class UploadFileAsNewTableHQ extends UploadFileHQ { + @Nonnull public final LinkedHashMap header; - - UploadFileAsNewTableHQ(String tableName, File file, LinkedHashMap header) { + UploadFileAsNewTableHQ(@Nonnull final String tableName, @Nonnull final File file, + @Nonnull final LinkedHashMap header) { super(tableName, file); - Preconditions.checkArgument(file.exists(), "%s is not found", file.getPath()); - this.header = header; } } diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java index 209864eb..378521df 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java @@ -20,6 +20,7 @@ import hivemall.utils.lang.Preconditions; +import javax.annotation.Nonnull; import java.io.File; public abstract class UploadFileHQ extends TableHQ { @@ -27,22 +28,22 @@ public enum Format { MSGPACK, CSV, TSV, UNKNOWN } + @Nonnull public final File file; + @Nonnull public final Format format; - - UploadFileHQ(String tableName, File file) { + UploadFileHQ(@Nonnull final String tableName, @Nonnull final File file) { super(tableName); - Preconditions.checkArgument(file.exists(), "%s is not found", file.getPath()); + Preconditions.checkArgument(file.exists(), "%s not found", file.getPath()); this.file = file; this.format = guessFormat(file); } - - Format guessFormat(File file) { - String fileName = file.getName(); + private Format guessFormat(File file) { + final String fileName = file.getName(); if (fileName.endsWith(".msgpack.gz")) { return Format.MSGPACK; } else if (fileName.endsWith(".csv")) { diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java index 16fe3b9c..fc7873ec 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java @@ -18,10 +18,11 @@ */ package hivemall.systemtest.model; +import javax.annotation.Nonnull; import java.io.File; public class UploadFileToExistingHQ extends UploadFileHQ { - UploadFileToExistingHQ(String tableName, File file) { + UploadFileToExistingHQ(@Nonnull final String tableName, @Nonnull final File file) { super(tableName, file); } } diff --git a/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java index 0d961ec3..72399d6b 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java @@ -23,34 +23,41 @@ import hivemall.systemtest.model.HQ; import hivemall.systemtest.model.RawHQ; import hivemall.systemtest.utils.IO; +import hivemall.utils.lang.Preconditions; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; public class LazyMatchingResource { - private String fileName; - private Charset charset; + @Nonnull + private final String fileName; + @Nonnull + private final Charset charset; - - public LazyMatchingResource(String fileName, Charset charset) { + public LazyMatchingResource(@Nonnull final String fileName, @Nonnull final Charset charset) { this.fileName = fileName; this.charset = charset; } + public List toStrict(@CheckForNull final String caseDir) { + Preconditions.checkNotNull(caseDir, "caseDir"); - public List toStrict(String caseDir) { - String query = IO.getFromResourcePath(caseDir + fileName, charset); - String formatted = CommandShellEmulation.HIVE_CLI.transformScript(query); - List split = StatementsSplitter.splitStatements(formatted); - List results = new ArrayList(); + final String query = IO.getFromResourcePath(caseDir + fileName, charset); + final String formatted = CommandShellEmulation.HIVE_CLI.transformScript(query); + final List split = StatementsSplitter.splitStatements(formatted); + final List results = new ArrayList(); for (String q : split) { results.add(HQ.fromStatement(q)); } return results; } - public String[] getAnswers(String answerDir) { + public String[] getAnswers(@CheckForNull final String answerDir) { + Preconditions.checkNotNull(answerDir, "answerDir"); + return IO.getFromResourcePath(answerDir + fileName).split(IO.QD); } } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java index 931b5a22..09d242dc 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java @@ -29,6 +29,7 @@ import hivemall.systemtest.model.UploadFileToExistingHQ; import org.junit.rules.TemporaryFolder; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -39,18 +40,16 @@ public class HiveSystemTestRunner extends SystemTestRunner { private TemporaryFolder tmpFolder; private HiveShell hShell; - - public HiveSystemTestRunner(SystemTestCommonInfo ci, String propertiesFile) { + public HiveSystemTestRunner(final SystemTestCommonInfo ci, final String propertiesFile) { super(ci, propertiesFile); } - public HiveSystemTestRunner(SystemTestCommonInfo ci) { + public HiveSystemTestRunner(final SystemTestCommonInfo ci) { super(ci, "hiverunner.properties"); } - @Override - protected void initRunner() { + void initRunner() { try { tmpFolder = new TemporaryFolder() { { @@ -64,20 +63,25 @@ protected void initRunner() { setHiveExecutionEngine(props.getProperty("hive.execution.engine", "mr")); // optional - if (props.containsKey("enableTimeout")) + if (props.containsKey("enableTimeout")) { setTimeoutEnabled(Boolean.valueOf(props.getProperty("enableTimeout"))); - if (props.containsKey("timeoutRetryLimit")) + } + if (props.containsKey("timeoutRetryLimit")) { setTimeoutRetries(Integer.valueOf(props.getProperty("timeoutRetryLimit"))); - if (props.containsKey("timeoutSeconds")) + } + if (props.containsKey("timeoutSeconds")) { setTimeoutSeconds(Integer.valueOf(props.getProperty("timeoutSeconds"))); - if (props.containsKey("commandShellEmulation")) + } + if (props.containsKey("commandShellEmulation")) { setCommandShellEmulation(CommandShellEmulation.valueOf(props.getProperty("commandShellEmulation"))); + } } }; - HiveServerContext ctx = Extractor.getStandaloneHiveServerContext(tmpFolder, config); + final HiveServerContext ctx = Extractor.getStandaloneHiveServerContext(tmpFolder, + config); container = Extractor.getHiveServerContainer(ctx); @SuppressWarnings("serial") - HiveShellBuilder builder = new HiveShellBuilder() { + final HiveShellBuilder builder = new HiveShellBuilder() { { putAllProperties(new HashMap() { { @@ -98,21 +102,23 @@ protected void initRunner() { @Override protected void finRunner() { - if (container != null) + if (container != null) { container.tearDown(); - if (tmpFolder != null) + } + if (tmpFolder != null) { tmpFolder.delete(); + } } @Override - protected List exec(RawHQ hq) { - logger.info("executing: `" + hq.get() + "`"); + protected List exec(@Nonnull final RawHQ hq) { + logger.info("executing: `" + hq.query + "`"); - return hShell.executeQuery(hq.get()); + return hShell.executeQuery(hq.query); } @Override - List uploadFileToExisting(final UploadFileToExistingHQ hq) throws Exception { + List uploadFileToExisting(@Nonnull final UploadFileToExistingHQ hq) throws Exception { logger.info("executing: insert " + hq.file.getPath() + " into " + hq.tableName + " on " + dbName); diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java index 12229d80..5b55466b 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java @@ -18,19 +18,25 @@ */ package hivemall.systemtest.runner; +import javax.annotation.Nonnull; + public class SystemTestCommonInfo { + @Nonnull public final String baseDir; + @Nonnull public final String caseDir; + @Nonnull public final String answerDir; + @Nonnull public final String initDir; + @Nonnull public final String dbName; - - public SystemTestCommonInfo(Class clazz) { + public SystemTestCommonInfo(@Nonnull final Class clazz) { baseDir = clazz.getName().replace(".", "/"); caseDir = baseDir + "/case/"; answerDir = baseDir + "/answer/"; initDir = baseDir + "/init/"; - dbName = clazz.getName().replace(".", "_"); + dbName = clazz.getName().replace(".", "_").toLowerCase(); } } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index 93bf6688..0854e319 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -19,6 +19,7 @@ package hivemall.systemtest.runner; import com.google.common.io.Resources; +import hivemall.systemtest.exception.QueryExecutionException; import hivemall.systemtest.model.CreateTableHQ; import hivemall.systemtest.model.DropTableHQ; import hivemall.systemtest.model.HQ; @@ -31,12 +32,16 @@ import hivemall.systemtest.model.UploadFileHQ; import hivemall.systemtest.model.UploadFileToExistingHQ; import hivemall.systemtest.utils.IO; +import hivemall.utils.lang.Preconditions; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.rules.ExternalResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.FileInputStream; import java.io.InputStream; import java.util.ArrayList; @@ -49,14 +54,22 @@ public abstract class SystemTestRunner extends ExternalResource { static final Logger logger = LoggerFactory.getLogger(SystemTestRunner.class); - List classInitHqs = new ArrayList(); - Set immutableTables = new HashSet(); + @Nonnull + final List classInitHqs; + @Nonnull + final Set immutableTables; + @Nonnull final String dbName; + @Nonnull final Properties props; + SystemTestRunner(@CheckForNull SystemTestCommonInfo ci, @CheckForNull String propertiesFile) { + Preconditions.checkNotNull(ci, "ci"); + Preconditions.checkNotNull(propertiesFile, "propertiesFile"); - SystemTestRunner(SystemTestCommonInfo ci, String propertiesFile) { - this.dbName = formatDBName(ci.dbName); + classInitHqs = new ArrayList(); + immutableTables = new HashSet(); + dbName = ci.dbName; final String path = "hivemall/" + propertiesFile; try { @@ -66,16 +79,16 @@ public abstract class SystemTestRunner extends ExternalResource { is = new FileInputStream(Resources.getResource(path).getPath()); props.load(is); } finally { - if (is != null) + if (is != null) { is.close(); + } } } catch (Exception ex) { - throw new RuntimeException("Failed to load properties from " + path + ". " + throw new IllegalArgumentException("Failed to load properties from " + path + ". " + ex.getMessage()); } } - @Override protected void before() throws Exception { initRunner(); @@ -87,7 +100,8 @@ protected void after() { try { resetDB(); // clean up database } catch (Exception ex) { - throw new RuntimeException("Failed to clean up temporary database. " + ex.getMessage()); + throw new QueryExecutionException("Failed to clean up temporary database. " + + ex.getMessage()); } finally { finRunner(); } @@ -97,16 +111,16 @@ protected void after() { abstract void finRunner(); - public void initBy(StrictHQ hq) { + public void initBy(@Nonnull final StrictHQ hq) { classInitHqs.add(hq); } - public void initBy(List hqs) { + public void initBy(@Nonnull final List hqs) { classInitHqs.addAll(hqs); } // fix to temporary database and user-defined init (should be called per Test class) - final void prepareDB() throws Exception { + void prepareDB() throws Exception { createDB(dbName); use(dbName); for (StrictHQ q : classInitHqs) { @@ -122,59 +136,61 @@ final void prepareDB() throws Exception { } // drop temporary database (should be called per Test class) - final void resetDB() throws Exception { + void resetDB() throws Exception { dropDB(dbName); } - String formatDBName(String dbName) { - return dbName; - } - - public final boolean isImmutableTable(String tableName) { + public final boolean isImmutableTable(final String tableName) { return immutableTables.contains(tableName); } // execute StrictHQ - public List exec(StrictHQ hq) throws Exception { - if (hq instanceof RawHQ) + public List exec(@Nonnull final StrictHQ hq) throws Exception { + if (hq instanceof RawHQ) { return exec((RawHQ) hq); - else if (hq instanceof TableHQ) + } else if (hq instanceof TableHQ) { return exec((TableHQ) hq); - else if (hq instanceof TableListHQ) + } else if (hq instanceof TableListHQ) { return tableList(); - else - throw new RuntimeException("Unexpected query type: " + hq.getClass()); + } else { + throw new IllegalArgumentException("Unexpected query type: " + hq.getClass()); + } } //// execute RawHQ - abstract protected List exec(RawHQ hq) throws Exception; + abstract protected List exec(@Nonnull final RawHQ hq) throws Exception; //// execute TableHQ - protected List exec(TableHQ hq) throws Exception { - if (hq instanceof CreateTableHQ) + List exec(@Nonnull final TableHQ hq) throws Exception { + if (hq instanceof CreateTableHQ) { return createTable((CreateTableHQ) hq); - else if (hq instanceof DropTableHQ) + } else if (hq instanceof DropTableHQ) { return dropTable((DropTableHQ) hq); - else if (hq instanceof InsertHQ) + } else if (hq instanceof InsertHQ) { return insert((InsertHQ) hq); - else if (hq instanceof UploadFileHQ) + } else if (hq instanceof UploadFileHQ) { return exec((UploadFileHQ) hq); - else - throw new RuntimeException("Unexpected query type: " + hq.getClass()); + } else { + throw new IllegalArgumentException("Unexpected query type: " + hq.getClass()); + } } ////// execute UploadFileHQ - protected List exec(UploadFileHQ hq) throws Exception { - if (hq instanceof UploadFileAsNewTableHQ) + List exec(@Nonnull final UploadFileHQ hq) throws Exception { + if (hq instanceof UploadFileAsNewTableHQ) { return uploadFileAsNewTable((UploadFileAsNewTableHQ) hq); - else if (hq instanceof UploadFileToExistingHQ) + } else if (hq instanceof UploadFileToExistingHQ) { return uploadFileToExisting((UploadFileToExistingHQ) hq); - else - throw new RuntimeException("Unexpected query type: " + hq.getClass()); + } else { + throw new IllegalArgumentException("Unexpected query type: " + hq.getClass()); + } } // matching StrictHQ - public void matching(StrictHQ hq, String answer, boolean ordered) throws Exception { + public void matching(@Nonnull final StrictHQ hq, @CheckForNull final String answer, + final boolean ordered) throws Exception { + Preconditions.checkNotNull(answer, "answer"); + List result = exec(hq); if (ordered) { @@ -189,23 +205,24 @@ public void matching(StrictHQ hq, String answer, boolean ordered) throws Excepti } // matching StrictHQ (ordered == false) - public void matching(StrictHQ hq, String answer) throws Exception { + public void matching(@Nonnull final StrictHQ hq, @CheckForNull final String answer) + throws Exception { matching(hq, answer, false); } - List createDB(String dbName) throws Exception { + List createDB(@Nonnull final String dbName) throws Exception { logger.info("executing: create database if not exists" + dbName); return exec(HQ.fromStatement("CREATE DATABASE IF NOT EXISTS " + dbName)); } - List dropDB(String dbName) throws Exception { + List dropDB(@Nonnull final String dbName) throws Exception { logger.info("executing: drop database if exists " + dbName); return exec(HQ.fromStatement("DROP DATABASE IF EXISTS " + dbName + " CASCADE")); } - List use(String dbName) throws Exception { + List use(@Nonnull final String dbName) throws Exception { logger.info("executing: use " + dbName); return exec(HQ.fromStatement("USE " + dbName)); @@ -217,20 +234,20 @@ List tableList() throws Exception { return exec(HQ.fromStatement("SHOW TABLES")); } - List createTable(CreateTableHQ hq) throws Exception { + List createTable(@Nonnull final CreateTableHQ hq) throws Exception { logger.info("executing: create table " + hq.tableName + " if not exists on " + dbName); return exec(HQ.fromStatement("CREATE TABLE IF NOT EXISTS " + hq.tableName + hq.getTableDeclaration())); } - List dropTable(DropTableHQ hq) throws Exception { + List dropTable(@Nonnull final DropTableHQ hq) throws Exception { logger.info("executing: drop table " + hq.tableName + " if exists on " + dbName); return exec(HQ.fromStatement("DROP TABLE IF EXISTS " + hq.tableName)); } - List insert(InsertHQ hq) throws Exception { + List insert(@Nonnull final InsertHQ hq) throws Exception { logger.info("executing: insert into " + hq.tableName + " on " + dbName); // *WORKAROUND* @@ -240,7 +257,7 @@ List insert(InsertHQ hq) throws Exception { // `WITH ... AS (SELECT ...) INSERT INTO TABLE ... SELECT * FROM ...` // can insert anything on hiverunner(v3.0.0) // cannot use map on TD(v20160901) - StringBuilder sb = new StringBuilder(); + final StringBuilder sb = new StringBuilder(); sb.append("WITH temporary_table_for_with_clause AS ("); for (Object[] row : hq.data) { sb.append("SELECT "); @@ -261,7 +278,7 @@ List insert(InsertHQ hq) throws Exception { return exec(HQ.fromStatement(sb.toString())); } - List uploadFileAsNewTable(UploadFileAsNewTableHQ hq) throws Exception { + List uploadFileAsNewTable(@Nonnull final UploadFileAsNewTableHQ hq) throws Exception { logger.info("executing: create " + hq.tableName + " based on " + hq.file.getPath() + " if not exists on " + dbName); @@ -269,15 +286,16 @@ List uploadFileAsNewTable(UploadFileAsNewTableHQ hq) throws Exception { return uploadFileToExisting(HQ.uploadByFullPathToExisting(hq.tableName, hq.file.getPath())); } - abstract List uploadFileToExisting(UploadFileToExistingHQ hq) throws Exception; + abstract List uploadFileToExisting(@Nonnull final UploadFileToExistingHQ hq) + throws Exception; - private String serialize(Object val) { + private String serialize(@Nullable final Object val) { // NOTE: this method is low-performance, don't use w/ big data if (val instanceof String) { return "'" + String.valueOf(val) + "'"; } else if (val instanceof Object[]) { - Object[] objs = (Object[]) val; - StringBuilder sb = new StringBuilder(); + final Object[] objs = (Object[]) val; + final StringBuilder sb = new StringBuilder(); sb.append("array("); for (Object o : objs) { sb.append(serialize(o)); @@ -287,8 +305,8 @@ private String serialize(Object val) { sb.append(")"); return sb.toString(); } else if (val instanceof List) { - List list = (List) val; - StringBuilder sb = new StringBuilder(); + final List list = (List) val; + final StringBuilder sb = new StringBuilder(); sb.append("array("); for (Object o : list) { sb.append(serialize(o)); @@ -298,8 +316,8 @@ private String serialize(Object val) { sb.append(")"); return sb.toString(); } else if (val instanceof Map) { - Map map = (Map) val; - StringBuilder sb = new StringBuilder(); + final Map map = (Map) val; + final StringBuilder sb = new StringBuilder(); sb.append("map("); for (Map.Entry e : map.entrySet()) { sb.append(serialize(e.getKey())); diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index 54090862..1369f741 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -18,6 +18,7 @@ */ package hivemall.systemtest.runner; +import hivemall.systemtest.exception.QueryExecutionException; import hivemall.systemtest.model.HQ; import hivemall.systemtest.model.RawHQ; import hivemall.systemtest.model.StrictHQ; @@ -25,6 +26,8 @@ import hivemall.utils.lang.Preconditions; import org.junit.rules.ExternalResource; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Arrays; @@ -34,20 +37,27 @@ import java.util.Map.Entry; public class SystemTestTeam extends ExternalResource { - private List runners = new ArrayList(); - private List reachGoal = new ArrayList(); // distinct + @Nonnull + private final List runners; + @Nonnull + private final List reachGoal; - private List initHqs = new ArrayList(); - private Map, Boolean> entries = new LinkedHashMap, Boolean>(); + @Nonnull + private final List initHqs; + @Nonnull + private final Map, Boolean> entries; private boolean needRun = false; // remind `run()` + public SystemTestTeam(final SystemTestRunner... runners) { + this.runners = new ArrayList(); + this.reachGoal = new ArrayList(); // distinct + this.initHqs = new ArrayList(); + this.entries = new LinkedHashMap, Boolean>(); - public SystemTestTeam(SystemTestRunner... runners) { this.runners.addAll(Arrays.asList(runners)); } - @Override protected void after() { if (needRun) { @@ -56,49 +66,57 @@ protected void after() { for (SystemTestRunner runner : reachGoal) { try { - List tables = runner.exec(HQ.tableList()); + final List tables = runner.exec(HQ.tableList()); for (String t : tables) { if (!runner.isImmutableTable(t)) { runner.exec(HQ.dropTable(t)); } } } catch (Exception ex) { - throw new RuntimeException("Failed to resetPerMethod database. " + ex.getMessage()); + throw new QueryExecutionException("Failed to resetPerMethod database. " + + ex.getMessage()); } } } // add additional runner for each @Test method - public void add(SystemTestRunner... runners) { + public void add(final SystemTestRunner... runners) { this.runners.addAll(Arrays.asList(runners)); } // add initialization for each @Test method - public void initBy(StrictHQ hq) { + public void initBy(@Nonnull final StrictHQ hq) { initHqs.add(hq); needRun = true; } - public void initBy(List hqs) { + public void initBy(@Nonnull final List hqs) { initHqs.addAll(hqs); needRun = true; } - public void set(StrictHQ hq, String expected, boolean ordered) { + public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected, boolean ordered) { + Preconditions.checkNotNull(expected, "expected"); + entries.put(pair(hq, expected), ordered); needRun = true; } - public void set(StrictHQ hq, String expected) { + public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected) { + Preconditions.checkNotNull(expected, "expected"); + entries.put(pair(hq, expected), false); needRun = true; } - public void set(List hqs, List expecteds, List ordereds) { + public void set(@Nonnull final List hqs, + @CheckForNull final List expecteds, @CheckForNull final List ordereds) { + Preconditions.checkNotNull(expecteds, "expecteds"); + Preconditions.checkNotNull(ordereds, "ordereds"); Preconditions.checkArgument(hqs.size() == expecteds.size(), "Mismatch between number of queries(%s) and length of answers(%s)", hqs.size(), expecteds.size()); @@ -113,17 +131,19 @@ public void set(List hqs, List expecteds, List hqs, List expecteds) { - List ordereds = new ArrayList(); + public void set(@Nonnull final List hqs, + @CheckForNull final List expecteds) { + final List ordereds = new ArrayList(); for (int i = 0; i < hqs.size(); i++) { ordereds.add(false); } set(hqs, expecteds, ordereds); } - public void set(LazyMatchingResource hq, SystemTestCommonInfo ci, boolean ordered) { - List rhqs = hq.toStrict(ci.caseDir); - String[] answers = hq.getAnswers(ci.answerDir); + public void set(@Nonnull final LazyMatchingResource hq, + @CheckForNull final SystemTestCommonInfo ci, final boolean ordered) { + final List rhqs = hq.toStrict(ci.caseDir); + final String[] answers = hq.getAnswers(ci.answerDir); Preconditions.checkArgument(rhqs.size() == answers.length, "Mismatch between number of queries(%s) and length of answers(%s)", rhqs.size(), @@ -136,7 +156,7 @@ public void set(LazyMatchingResource hq, SystemTestCommonInfo ci, boolean ordere needRun = true; } - public void set(LazyMatchingResource hq, SystemTestCommonInfo ci) { + public void set(@Nonnull final LazyMatchingResource hq, final SystemTestCommonInfo ci) { set(hq, ci, false); } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index 6c381a70..388e3196 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -29,6 +29,7 @@ import com.treasuredata.client.model.TDResultFormat; import com.treasuredata.client.model.TDTable; import hivemall.systemtest.ConvertToMsgpack; +import hivemall.systemtest.exception.QueryExecutionException; import hivemall.systemtest.model.CreateTableHQ; import hivemall.systemtest.model.DropTableHQ; import hivemall.systemtest.model.HQ; @@ -38,6 +39,7 @@ import hivemall.systemtest.utils.IO; import org.apache.commons.csv.CSVFormat; +import javax.annotation.Nonnull; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -59,34 +61,31 @@ public class TDSystemTestRunner extends SystemTestRunner { private int fileUploadCommitBackOff = 5; private int fileUploadCommitRetryLimit = 7; - - public TDSystemTestRunner(SystemTestCommonInfo ci, String propertiesFile) { + public TDSystemTestRunner(final SystemTestCommonInfo ci, final String propertiesFile) { super(ci, propertiesFile); } - public TDSystemTestRunner(SystemTestCommonInfo ci) { + public TDSystemTestRunner(final SystemTestCommonInfo ci) { super(ci, "td.properties"); } - - @Override - String formatDBName(String name) { - return name.toLowerCase(); - } - @Override protected void initRunner() { // optional - if (props.containsKey("execFinishRetryLimit")) + if (props.containsKey("execFinishRetryLimit")) { execFinishRetryLimit = Integer.valueOf(props.getProperty("execFinishRetryLimit")); - if (props.containsKey("fileUploadPerformRetryLimit")) + } + if (props.containsKey("fileUploadPerformRetryLimit")) { fileUploadPerformRetryLimit = Integer.valueOf(props.getProperty("fileUploadPerformRetryLimit")); - if (props.containsKey("fileUploadCommitBackOff")) + } + if (props.containsKey("fileUploadCommitBackOff")) { fileUploadCommitBackOff = Integer.valueOf(props.getProperty("fileUploadCommitBackOff")); - if (props.containsKey("fileUploadCommitRetryLimit")) + } + if (props.containsKey("fileUploadCommitRetryLimit")) { fileUploadCommitRetryLimit = Integer.valueOf(props.getProperty("fileUploadCommitRetryLimit")); + } - Properties TDPorps = System.getProperties(); + final Properties TDPorps = System.getProperties(); for (Map.Entry e : props.entrySet()) { if (e.getKey().toString().startsWith("td.client.")) { TDPorps.setProperty(e.getKey().toString(), e.getValue().toString()); @@ -100,18 +99,19 @@ protected void initRunner() { @Override protected void finRunner() { - if (client != null) + if (client != null) { client.close(); + } } @Override - protected List exec(RawHQ hq) throws Exception { - logger.info("executing: `" + hq.get() + "`"); + protected List exec(@Nonnull final RawHQ hq) throws Exception { + logger.info("executing: `" + hq.query + "`"); - TDJobRequest req = TDJobRequest.newHiveQuery(dbName, hq.get()); - String id = client.submit(req); + final TDJobRequest req = TDJobRequest.newHiveQuery(dbName, hq.query); + final String id = client.submit(req); - ExponentialBackOff backOff = new ExponentialBackOff(); + final ExponentialBackOff backOff = new ExponentialBackOff(); TDJobSummary job = client.jobStatus(id); int nRetries = 0; while (!job.getStatus().isFinished()) { @@ -129,7 +129,7 @@ protected List exec(RawHQ hq) throws Exception { return client.jobResult(id, TDResultFormat.TSV, new Function>() { @Override public List apply(InputStream input) { - List results = new ArrayList(); + final List results = new ArrayList(); BufferedReader reader = null; try { try { @@ -139,11 +139,13 @@ public List apply(InputStream input) { results.addAll(Arrays.asList(line.split(IO.RD))); } } finally { - if (reader != null) + if (reader != null) { reader.close(); + } } } catch (IOException ex) { - throw new RuntimeException("Failed to read results from TD. " + ex.getMessage()); + throw new QueryExecutionException("Failed to read results from TD. " + + ex.getMessage()); } return results; } @@ -151,7 +153,7 @@ public List apply(InputStream input) { } @Override - protected List createDB(final String dbName) throws Exception { + protected List createDB(@Nonnull final String dbName) throws Exception { logger.info("executing: create database if not exists " + dbName); client.createDatabaseIfNotExists(dbName); @@ -159,7 +161,7 @@ protected List createDB(final String dbName) throws Exception { } @Override - protected List dropDB(final String dbName) throws Exception { + protected List dropDB(@Nonnull final String dbName) throws Exception { logger.info("executing: drop database if exists " + dbName); client.deleteDatabaseIfExists(dbName); @@ -167,7 +169,7 @@ protected List dropDB(final String dbName) throws Exception { } @Override - protected List use(final String dbName) throws Exception { + protected List use(@Nonnull final String dbName) throws Exception { return Collections.singletonList("No need to execute `USE` statement on TD, so skipped `USE " + dbName + "`"); } @@ -176,8 +178,8 @@ protected List use(final String dbName) throws Exception { protected List tableList() throws Exception { logger.info("executing: show tables on " + dbName); - List tables = client.listTables(dbName); - List result = new ArrayList(); + final List tables = client.listTables(dbName); + final List result = new ArrayList(); for (TDTable t : tables) { result.add(t.getName()); } @@ -185,10 +187,10 @@ protected List tableList() throws Exception { } @Override - protected List createTable(final CreateTableHQ hq) throws Exception { + protected List createTable(@Nonnull final CreateTableHQ hq) throws Exception { logger.info("executing: create table " + hq.tableName + " if not exists on " + dbName); - List columns = new ArrayList(); + final List columns = new ArrayList(); for (Map.Entry e : hq.header.entrySet()) { columns.add(new TDColumn(e.getKey(), TDColumnType.parseColumnType(e.getValue()))); } @@ -198,7 +200,7 @@ protected List createTable(final CreateTableHQ hq) throws Exception { } @Override - protected List dropTable(final DropTableHQ hq) throws Exception { + protected List dropTable(@Nonnull final DropTableHQ hq) throws Exception { logger.info("executing: drop table " + hq.tableName + " if exists on " + dbName); client.deleteTableIfExists(dbName, hq.tableName); @@ -206,14 +208,15 @@ protected List dropTable(final DropTableHQ hq) throws Exception { } @Override - protected List uploadFileAsNewTable(final UploadFileAsNewTableHQ hq) throws Exception { + protected List uploadFileAsNewTable(@Nonnull final UploadFileAsNewTableHQ hq) + throws Exception { logger.info("executing: create " + hq.tableName + " based on " + hq.file.getPath() + " if not exists on " + dbName); createTable(HQ.createTable(hq.tableName, hq.header)); - String sessionName = "session-" + String.valueOf(System.currentTimeMillis()); - String partName = "part-of-" + String.valueOf(sessionName); + final String sessionName = "session-" + String.valueOf(System.currentTimeMillis()); + final String partName = "part-of-" + String.valueOf(sessionName); client.createBulkImportSession(sessionName, dbName, hq.tableName); try { @@ -223,7 +226,7 @@ protected List uploadFileAsNewTable(final UploadFileAsNewTableHQ hq) thr client.uploadBulkImportPart(sessionName, partName, hq.file); break; case CSV: { - File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + final File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); to.deleteOnExit(); client.uploadBulkImportPart(sessionName, partName, @@ -232,7 +235,7 @@ protected List uploadFileAsNewTable(final UploadFileAsNewTableHQ hq) thr break; } case TSV: { - File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + final File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); to.deleteOnExit(); client.uploadBulkImportPart(sessionName, partName, @@ -246,7 +249,7 @@ protected List uploadFileAsNewTable(final UploadFileAsNewTableHQ hq) thr client.freezeBulkImportSession(sessionName); client.performBulkImportSession(sessionName); - ExponentialBackOff backOff = new ExponentialBackOff(); + final ExponentialBackOff backOff = new ExponentialBackOff(); TDBulkImportSession session = client.getBulkImportSession(sessionName); int performNRetries = 0; while (session.getStatus() == TDBulkImportSession.ImportStatus.PERFORMING) { @@ -285,12 +288,13 @@ protected List uploadFileAsNewTable(final UploadFileAsNewTableHQ hq) thr } @Override - protected List uploadFileToExisting(final UploadFileToExistingHQ hq) throws Exception { + protected List uploadFileToExisting(@Nonnull final UploadFileToExistingHQ hq) + throws Exception { logger.info("executing: insert " + hq.file.getPath() + " into " + hq.tableName + " on " + dbName); - String sessionName = "session-" + String.valueOf(System.currentTimeMillis()); - String partName = "part-of-" + String.valueOf(sessionName); + final String sessionName = "session-" + String.valueOf(System.currentTimeMillis()); + final String partName = "part-of-" + String.valueOf(sessionName); client.createBulkImportSession(sessionName, dbName, hq.tableName); try { @@ -319,7 +323,7 @@ protected List uploadFileToExisting(final UploadFileToExistingHQ hq) thr client.freezeBulkImportSession(sessionName); client.performBulkImportSession(sessionName); - ExponentialBackOff backOff = new ExponentialBackOff(); + final ExponentialBackOff backOff = new ExponentialBackOff(); TDBulkImportSession session = client.getBulkImportSession(sessionName); while (session.getStatus() == TDBulkImportSession.ImportStatus.PERFORMING) { logger.debug("Waiting bulk import completion"); @@ -341,8 +345,8 @@ protected List uploadFileToExisting(final UploadFileToExistingHQ hq) thr return Collections.singletonList("uploaded " + hq.file.getName() + " into " + hq.tableName); } - private List getHeaderFromTD(String tableName) { - List header = new ArrayList(); + private List getHeaderFromTD(@Nonnull final String tableName) { + final List header = new ArrayList(); for (TDTable t : client.listTables(dbName)) { if (t.getName().equals(tableName)) { List cols = t.getColumns(); diff --git a/systemtest/src/main/java/hivemall/systemtest/utils/IO.java b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java index f0767b71..dd133f8a 100644 --- a/systemtest/src/main/java/hivemall/systemtest/utils/IO.java +++ b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java @@ -21,6 +21,7 @@ import com.google.common.io.Resources; import hivemall.utils.lang.Preconditions; +import javax.annotation.CheckForNull; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -31,43 +32,47 @@ public class IO { public static final String RD = "\t"; // row delimiter public static final String QD = "\n"; // query delimiter - private IO() {} - - public static String getFromFullPath(String fullPath, Charset charset) { - Preconditions.checkArgument(new File(fullPath).exists(), "%s is not found", fullPath); + public static String getFromFullPath(@CheckForNull final String fullPath, final Charset charset) { + Preconditions.checkNotNull(fullPath, "fullPath"); return new String(readAllBytes(fullPath), charset); } - public static String getFromFullPath(String fullPath) { + public static String getFromFullPath(@CheckForNull final String fullPath) { return getFromFullPath(fullPath, Charset.defaultCharset()); } - public static String getFromResourcePath(String resourcePath, Charset charset) { - String fullPath = Resources.getResource(resourcePath).getPath(); + public static String getFromResourcePath(@CheckForNull final String resourcePath, + final Charset charset) { + Preconditions.checkNotNull(resourcePath, "resourcePath"); + + final String fullPath = Resources.getResource(resourcePath).getPath(); return getFromFullPath(fullPath, charset); } - public static String getFromResourcePath(String resourcePath) { + public static String getFromResourcePath(@CheckForNull final String resourcePath) { return getFromResourcePath(resourcePath, Charset.defaultCharset()); } - private static byte[] readAllBytes(String filePath) { - File f = new File(filePath); + private static byte[] readAllBytes(final String filePath) { + final File file = new File(filePath); + + Preconditions.checkArgument(file.exists(), "%s not found", filePath); - int len = (int) f.length(); - byte[] buf = new byte[len]; + final int len = (int) file.length(); + final byte[] buf = new byte[len]; InputStream is = null; try { try { - is = new FileInputStream(f); + is = new FileInputStream(file); is.read(buf); } finally { - if (is != null) + if (is != null) { is.close(); + } } } catch (IOException ex) { throw new RuntimeException("Failed to read " + filePath + ". " + ex.getMessage()); From 0b01896eed8411e36676f2d3924724b28a4ce345 Mon Sep 17 00:00:00 2001 From: amaya Date: Mon, 12 Sep 2016 15:22:45 +0900 Subject: [PATCH 17/28] avoid running with no runner --- .../main/java/hivemall/systemtest/runner/SystemTestTeam.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index 1369f741..5c6dd750 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -163,6 +163,10 @@ public void set(@Nonnull final LazyMatchingResource hq, final SystemTestCommonIn public void run() throws Exception { needRun = false; + if (runners.size() == 0) { + throw new IllegalStateException("Set at least one runner."); + } + for (SystemTestRunner runner : runners) { if (!reachGoal.contains(runner)) { // initialization each @Test methods From b5f35c3ea3512238fa1eac6cfe4a111a1469c754 Mon Sep 17 00:00:00 2001 From: amaya Date: Tue, 13 Sep 2016 16:03:32 +0900 Subject: [PATCH 18/28] refresh null check --- .../java/hivemall/systemtest/model/HQ.java | 33 ++++++++++--------- .../model/lazy/LazyMatchingResource.java | 4 +-- .../systemtest/runner/SystemTestRunner.java | 6 ++-- .../systemtest/runner/SystemTestTeam.java | 8 ++--- .../systemtest/runner/TDSystemTestRunner.java | 10 +++--- .../java/hivemall/systemtest/utils/IO.java | 4 +-- 6 files changed, 33 insertions(+), 32 deletions(-) diff --git a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java index 1e45016c..ee882561 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java @@ -32,10 +32,11 @@ import java.util.LinkedHashMap; import java.util.List; -abstract public class HQ { +public abstract class HQ { + @Nonnull public static RawHQ fromStatement(String query) { - Preconditions.checkNotNull(query, "query"); + Preconditions.checkNotNull(query); final String formatted = CommandShellEmulation.HIVE_CLI.transformScript(query); final List split = StatementsSplitter.splitStatements(formatted); @@ -50,7 +51,7 @@ public static RawHQ fromStatement(String query) { @Nonnull public static List fromStatements(@CheckForNull final String queries) { - Preconditions.checkNotNull(queries, "queries"); + Preconditions.checkNotNull(queries); final String formatted = CommandShellEmulation.HIVE_CLI.transformScript(queries); final List split = StatementsSplitter.splitStatements(formatted); @@ -64,8 +65,8 @@ public static List fromStatements(@CheckForNull final String queries) { @Nonnull public static LazyMatchingResource autoMatchingByFileName(@CheckForNull final String fileName, @CheckForNull final Charset charset) { - Preconditions.checkNotNull(fileName, "fileName"); - Preconditions.checkNotNull(charset, "charset"); + Preconditions.checkNotNull(fileName); + Preconditions.checkNotNull(charset); return new LazyMatchingResource(fileName, charset); } @@ -93,15 +94,15 @@ public static TableListHQ tableList() { @Nonnull public static CreateTableHQ createTable(@CheckForNull final String tableName, @CheckForNull final LinkedHashMap header) { - Preconditions.checkNotNull(tableName, "tableName"); - Preconditions.checkNotNull(header, "header"); + Preconditions.checkNotNull(tableName); + Preconditions.checkNotNull(header); return new CreateTableHQ(tableName, header); } @Nonnull public static DropTableHQ dropTable(@CheckForNull final String tableName) { - Preconditions.checkNotNull(tableName, "tableName"); + Preconditions.checkNotNull(tableName); return new DropTableHQ(tableName); } @@ -109,9 +110,9 @@ public static DropTableHQ dropTable(@CheckForNull final String tableName) { @Nonnull public static InsertHQ insert(@CheckForNull final String tableName, @CheckForNull final List header, @CheckForNull final List data) { - Preconditions.checkNotNull(tableName, "tableName"); - Preconditions.checkNotNull(header, "header"); - Preconditions.checkNotNull(data, "data"); + Preconditions.checkNotNull(tableName); + Preconditions.checkNotNull(header); + Preconditions.checkNotNull(data); return new InsertHQ(tableName, header, data); } @@ -119,8 +120,8 @@ public static InsertHQ insert(@CheckForNull final String tableName, @Nonnull public static UploadFileToExistingHQ uploadByFullPathToExisting( @CheckForNull final String tableName, @CheckForNull final String fullPath) { - Preconditions.checkNotNull(tableName, "tableName"); - Preconditions.checkNotNull(fullPath, "fullPath"); + Preconditions.checkNotNull(tableName); + Preconditions.checkNotNull(fullPath); return new UploadFileToExistingHQ(tableName, new File(fullPath)); } @@ -135,9 +136,9 @@ public static UploadFileToExistingHQ uploadByResourcePathToExisting(final String public static UploadFileAsNewTableHQ uploadByFullPathAsNewTable( @CheckForNull final String tableName, @CheckForNull final String fullPath, @CheckForNull final LinkedHashMap header) { - Preconditions.checkNotNull(tableName, "tableName"); - Preconditions.checkNotNull(fullPath, "fullPath"); - Preconditions.checkNotNull(header, "header"); + Preconditions.checkNotNull(tableName); + Preconditions.checkNotNull(fullPath); + Preconditions.checkNotNull(header); final File file = new File(fullPath); diff --git a/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java index 72399d6b..3715a9e4 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java @@ -43,7 +43,7 @@ public LazyMatchingResource(@Nonnull final String fileName, @Nonnull final Chars } public List toStrict(@CheckForNull final String caseDir) { - Preconditions.checkNotNull(caseDir, "caseDir"); + Preconditions.checkNotNull(caseDir); final String query = IO.getFromResourcePath(caseDir + fileName, charset); final String formatted = CommandShellEmulation.HIVE_CLI.transformScript(query); @@ -56,7 +56,7 @@ public List toStrict(@CheckForNull final String caseDir) { } public String[] getAnswers(@CheckForNull final String answerDir) { - Preconditions.checkNotNull(answerDir, "answerDir"); + Preconditions.checkNotNull(answerDir); return IO.getFromResourcePath(answerDir + fileName).split(IO.QD); } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index 0854e319..b710f435 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -64,8 +64,8 @@ public abstract class SystemTestRunner extends ExternalResource { final Properties props; SystemTestRunner(@CheckForNull SystemTestCommonInfo ci, @CheckForNull String propertiesFile) { - Preconditions.checkNotNull(ci, "ci"); - Preconditions.checkNotNull(propertiesFile, "propertiesFile"); + Preconditions.checkNotNull(ci); + Preconditions.checkNotNull(propertiesFile); classInitHqs = new ArrayList(); immutableTables = new HashSet(); @@ -189,7 +189,7 @@ List exec(@Nonnull final UploadFileHQ hq) throws Exception { // matching StrictHQ public void matching(@Nonnull final StrictHQ hq, @CheckForNull final String answer, final boolean ordered) throws Exception { - Preconditions.checkNotNull(answer, "answer"); + Preconditions.checkNotNull(answer); List result = exec(hq); diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index 5c6dd750..c9291aa3 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -98,7 +98,7 @@ public void initBy(@Nonnull final List hqs) { } public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected, boolean ordered) { - Preconditions.checkNotNull(expected, "expected"); + Preconditions.checkNotNull(expected); entries.put(pair(hq, expected), ordered); @@ -106,7 +106,7 @@ public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected, } public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected) { - Preconditions.checkNotNull(expected, "expected"); + Preconditions.checkNotNull(expected); entries.put(pair(hq, expected), false); @@ -115,8 +115,8 @@ public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected) public void set(@Nonnull final List hqs, @CheckForNull final List expecteds, @CheckForNull final List ordereds) { - Preconditions.checkNotNull(expecteds, "expecteds"); - Preconditions.checkNotNull(ordereds, "ordereds"); + Preconditions.checkNotNull(expecteds); + Preconditions.checkNotNull(ordereds); Preconditions.checkArgument(hqs.size() == expecteds.size(), "Mismatch between number of queries(%s) and length of answers(%s)", hqs.size(), expecteds.size()); diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index 388e3196..d646e7ee 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -28,7 +28,7 @@ import com.treasuredata.client.model.TDJobSummary; import com.treasuredata.client.model.TDResultFormat; import com.treasuredata.client.model.TDTable; -import hivemall.systemtest.ConvertToMsgpack; +import hivemall.systemtest.MsgpackConverter; import hivemall.systemtest.exception.QueryExecutionException; import hivemall.systemtest.model.CreateTableHQ; import hivemall.systemtest.model.DropTableHQ; @@ -230,7 +230,7 @@ protected List uploadFileAsNewTable(@Nonnull final UploadFileAsNewTableH to.deleteOnExit(); client.uploadBulkImportPart(sessionName, partName, - new ConvertToMsgpack(hq.file, new ArrayList(hq.header.keySet()), + new MsgpackConverter(hq.file, new ArrayList(hq.header.keySet()), CSVFormat.DEFAULT).asFile(to)); break; } @@ -239,7 +239,7 @@ protected List uploadFileAsNewTable(@Nonnull final UploadFileAsNewTableH to.deleteOnExit(); client.uploadBulkImportPart(sessionName, partName, - new ConvertToMsgpack(hq.file, new ArrayList(hq.header.keySet()), + new MsgpackConverter(hq.file, new ArrayList(hq.header.keySet()), CSVFormat.TDF).asFile(to)); break; } @@ -306,14 +306,14 @@ protected List uploadFileToExisting(@Nonnull final UploadFileToExistingH case CSV: { File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); to.deleteOnExit(); - client.uploadBulkImportPart(sessionName, partName, new ConvertToMsgpack( + client.uploadBulkImportPart(sessionName, partName, new MsgpackConverter( hq.file, getHeaderFromTD(hq.tableName), CSVFormat.DEFAULT).asFile(to)); break; } case TSV: { File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); to.deleteOnExit(); - client.uploadBulkImportPart(sessionName, partName, new ConvertToMsgpack( + client.uploadBulkImportPart(sessionName, partName, new MsgpackConverter( hq.file, getHeaderFromTD(hq.tableName), CSVFormat.TDF).asFile(to)); break; } diff --git a/systemtest/src/main/java/hivemall/systemtest/utils/IO.java b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java index dd133f8a..d572ed0e 100644 --- a/systemtest/src/main/java/hivemall/systemtest/utils/IO.java +++ b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java @@ -35,7 +35,7 @@ public class IO { private IO() {} public static String getFromFullPath(@CheckForNull final String fullPath, final Charset charset) { - Preconditions.checkNotNull(fullPath, "fullPath"); + Preconditions.checkNotNull(fullPath); return new String(readAllBytes(fullPath), charset); } @@ -46,7 +46,7 @@ public static String getFromFullPath(@CheckForNull final String fullPath) { public static String getFromResourcePath(@CheckForNull final String resourcePath, final Charset charset) { - Preconditions.checkNotNull(resourcePath, "resourcePath"); + Preconditions.checkNotNull(resourcePath); final String fullPath = Resources.getResource(resourcePath).getPath(); return getFromFullPath(fullPath, charset); From d2f0370c273f47007196af2e538aafe0bdbaed8e Mon Sep 17 00:00:00 2001 From: amaya Date: Tue, 13 Sep 2016 16:26:56 +0900 Subject: [PATCH 19/28] mod name and inheritance structure --- ...rtToMsgpack.java => MsgpackConverter.java} | 12 ++++---- .../java/hivemall/systemtest/model/HQ.java | 7 ++++- .../model/{StrictHQ.java => HQBase.java} | 3 +- .../java/hivemall/systemtest/model/RawHQ.java | 2 +- .../hivemall/systemtest/model/TableHQ.java | 2 +- .../systemtest/model/TableListHQ.java | 2 +- .../systemtest/runner/SystemTestRunner.java | 24 +++++++-------- .../systemtest/runner/SystemTestTeam.java | 30 +++++++++---------- 8 files changed, 43 insertions(+), 39 deletions(-) rename systemtest/src/main/java/hivemall/systemtest/{ConvertToMsgpack.java => MsgpackConverter.java} (92%) rename systemtest/src/main/java/hivemall/systemtest/model/{StrictHQ.java => HQBase.java} (94%) diff --git a/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java b/systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java similarity index 92% rename from systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java rename to systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java index 8d3e9400..c6383e3b 100644 --- a/systemtest/src/main/java/hivemall/systemtest/ConvertToMsgpack.java +++ b/systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java @@ -36,7 +36,7 @@ import java.util.List; import java.util.zip.GZIPOutputStream; -public class ConvertToMsgpack { +public class MsgpackConverter { @Nonnull private final File file; @Nonnull @@ -44,11 +44,11 @@ public class ConvertToMsgpack { @Nonnull private final CSVFormat format; - public ConvertToMsgpack(@CheckForNull File file, @CheckForNull List header, + public MsgpackConverter(@CheckForNull File file, @CheckForNull List header, @CheckForNull CSVFormat format) { - Preconditions.checkNotNull(file, "file"); - Preconditions.checkNotNull(header, "header"); - Preconditions.checkNotNull(format, "format"); + Preconditions.checkNotNull(file); + Preconditions.checkNotNull(header); + Preconditions.checkNotNull(format); Preconditions.checkArgument(file.exists(), "%s not found", file.getPath()); this.file = file; @@ -93,7 +93,7 @@ public byte[] asByteArray() throws Exception { } public File asFile(@CheckForNull File to, final boolean needTimeColumn) throws Exception { - Preconditions.checkNotNull(to, "to"); + Preconditions.checkNotNull(to); Preconditions.checkArgument(to.exists(), "%s not found", to.getPath()); FileOutputStream os = null; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java index ee882561..f847dfe2 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java @@ -32,7 +32,12 @@ import java.util.LinkedHashMap; import java.util.List; -public abstract class HQ { +/** + * Domain-specific Hive Query Factory + */ +public class HQ { + + private HQ() {} @Nonnull public static RawHQ fromStatement(String query) { diff --git a/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/HQBase.java similarity index 94% rename from systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java rename to systemtest/src/main/java/hivemall/systemtest/model/HQBase.java index 921e1939..4212a5a0 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/StrictHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQBase.java @@ -18,6 +18,5 @@ */ package hivemall.systemtest.model; - -public abstract class StrictHQ extends HQ { +public interface HQBase { } diff --git a/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java index 5575509b..7671ca1c 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java @@ -20,7 +20,7 @@ import javax.annotation.Nonnull; -public class RawHQ extends StrictHQ { +public class RawHQ implements HQBase { @Nonnull public final String query; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java index e6b9aec3..8c988766 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java @@ -20,7 +20,7 @@ import javax.annotation.Nonnull; -public abstract class TableHQ extends StrictHQ { +public abstract class TableHQ implements HQBase { @Nonnull public final String tableName; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java index 4b47cde8..a4283bbb 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java @@ -18,6 +18,6 @@ */ package hivemall.systemtest.model; -public class TableListHQ extends StrictHQ { +public class TableListHQ implements HQBase { TableListHQ() {} } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index b710f435..e3d94120 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -23,9 +23,9 @@ import hivemall.systemtest.model.CreateTableHQ; import hivemall.systemtest.model.DropTableHQ; import hivemall.systemtest.model.HQ; +import hivemall.systemtest.model.HQBase; import hivemall.systemtest.model.InsertHQ; import hivemall.systemtest.model.RawHQ; -import hivemall.systemtest.model.StrictHQ; import hivemall.systemtest.model.TableHQ; import hivemall.systemtest.model.TableListHQ; import hivemall.systemtest.model.UploadFileAsNewTableHQ; @@ -55,7 +55,7 @@ public abstract class SystemTestRunner extends ExternalResource { static final Logger logger = LoggerFactory.getLogger(SystemTestRunner.class); @Nonnull - final List classInitHqs; + final List classInitHqs; @Nonnull final Set immutableTables; @Nonnull @@ -67,7 +67,7 @@ public abstract class SystemTestRunner extends ExternalResource { Preconditions.checkNotNull(ci); Preconditions.checkNotNull(propertiesFile); - classInitHqs = new ArrayList(); + classInitHqs = new ArrayList(); immutableTables = new HashSet(); dbName = ci.dbName; @@ -111,11 +111,11 @@ protected void after() { abstract void finRunner(); - public void initBy(@Nonnull final StrictHQ hq) { + public void initBy(@Nonnull final HQBase hq) { classInitHqs.add(hq); } - public void initBy(@Nonnull final List hqs) { + public void initBy(@Nonnull final List hqs) { classInitHqs.addAll(hqs); } @@ -123,7 +123,7 @@ public void initBy(@Nonnull final List hqs) { void prepareDB() throws Exception { createDB(dbName); use(dbName); - for (StrictHQ q : classInitHqs) { + for (HQBase q : classInitHqs) { exec(q); if (q instanceof CreateTableHQ) { @@ -144,8 +144,8 @@ public final boolean isImmutableTable(final String tableName) { return immutableTables.contains(tableName); } - // execute StrictHQ - public List exec(@Nonnull final StrictHQ hq) throws Exception { + // execute HQBase + public List exec(@Nonnull final HQBase hq) throws Exception { if (hq instanceof RawHQ) { return exec((RawHQ) hq); } else if (hq instanceof TableHQ) { @@ -186,8 +186,8 @@ List exec(@Nonnull final UploadFileHQ hq) throws Exception { } } - // matching StrictHQ - public void matching(@Nonnull final StrictHQ hq, @CheckForNull final String answer, + // matching HQBase + public void matching(@Nonnull final HQBase hq, @CheckForNull final String answer, final boolean ordered) throws Exception { Preconditions.checkNotNull(answer); @@ -204,8 +204,8 @@ public void matching(@Nonnull final StrictHQ hq, @CheckForNull final String answ } } - // matching StrictHQ (ordered == false) - public void matching(@Nonnull final StrictHQ hq, @CheckForNull final String answer) + // matching HQBase (ordered == false) + public void matching(@Nonnull final HQBase hq, @CheckForNull final String answer) throws Exception { matching(hq, answer, false); } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index c9291aa3..babbfbb7 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -20,8 +20,8 @@ import hivemall.systemtest.exception.QueryExecutionException; import hivemall.systemtest.model.HQ; +import hivemall.systemtest.model.HQBase; import hivemall.systemtest.model.RawHQ; -import hivemall.systemtest.model.StrictHQ; import hivemall.systemtest.model.lazy.LazyMatchingResource; import hivemall.utils.lang.Preconditions; import org.junit.rules.ExternalResource; @@ -43,17 +43,17 @@ public class SystemTestTeam extends ExternalResource { private final List reachGoal; @Nonnull - private final List initHqs; + private final List initHqs; @Nonnull - private final Map, Boolean> entries; + private final Map, Boolean> entries; private boolean needRun = false; // remind `run()` public SystemTestTeam(final SystemTestRunner... runners) { this.runners = new ArrayList(); this.reachGoal = new ArrayList(); // distinct - this.initHqs = new ArrayList(); - this.entries = new LinkedHashMap, Boolean>(); + this.initHqs = new ArrayList(); + this.entries = new LinkedHashMap, Boolean>(); this.runners.addAll(Arrays.asList(runners)); } @@ -85,19 +85,19 @@ public void add(final SystemTestRunner... runners) { } // add initialization for each @Test method - public void initBy(@Nonnull final StrictHQ hq) { + public void initBy(@Nonnull final HQBase hq) { initHqs.add(hq); needRun = true; } - public void initBy(@Nonnull final List hqs) { + public void initBy(@Nonnull final List hqs) { initHqs.addAll(hqs); needRun = true; } - public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected, boolean ordered) { + public void set(@Nonnull final HQBase hq, @CheckForNull final String expected, boolean ordered) { Preconditions.checkNotNull(expected); entries.put(pair(hq, expected), ordered); @@ -105,7 +105,7 @@ public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected, needRun = true; } - public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected) { + public void set(@Nonnull final HQBase hq, @CheckForNull final String expected) { Preconditions.checkNotNull(expected); entries.put(pair(hq, expected), false); @@ -113,7 +113,7 @@ public void set(@Nonnull final StrictHQ hq, @CheckForNull final String expected) needRun = true; } - public void set(@Nonnull final List hqs, + public void set(@Nonnull final List hqs, @CheckForNull final List expecteds, @CheckForNull final List ordereds) { Preconditions.checkNotNull(expecteds); Preconditions.checkNotNull(ordereds); @@ -131,7 +131,7 @@ public void set(@Nonnull final List hqs, needRun = true; } - public void set(@Nonnull final List hqs, + public void set(@Nonnull final List hqs, @CheckForNull final List expecteds) { final List ordereds = new ArrayList(); for (int i = 0; i < hqs.size(); i++) { @@ -170,20 +170,20 @@ public void run() throws Exception { for (SystemTestRunner runner : runners) { if (!reachGoal.contains(runner)) { // initialization each @Test methods - for (StrictHQ q : initHqs) { + for (HQBase q : initHqs) { runner.exec(q); } reachGoal.add(runner); } - for (Entry, Boolean> entry : entries.entrySet()) { + for (Entry, Boolean> entry : entries.entrySet()) { runner.matching(entry.getKey().getKey(), entry.getKey().getValue(), entry.getValue()); } } } - private Entry pair(StrictHQ hq, String answer) { - return new SimpleEntry(hq, answer); + private Entry pair(HQBase hq, String answer) { + return new SimpleEntry(hq, answer); } } From 62e3588b69cd15f6742ed2361e190e99b6403bbf Mon Sep 17 00:00:00 2001 From: amaya Date: Wed, 16 Nov 2016 15:23:49 +0900 Subject: [PATCH 20/28] Update license headers --- systemtest/README.md | 18 ++++++++++++ systemtest/pom.xml | 17 ++++++++++- .../java/com/klarna/hiverunner/Extractor.java | 28 +++++++++---------- .../hivemall/systemtest/MsgpackConverter.java | 28 +++++++++---------- .../exception/QueryExecutionException.java | 28 +++++++++---------- .../systemtest/model/CreateTableHQ.java | 28 +++++++++---------- .../systemtest/model/DropTableHQ.java | 28 +++++++++---------- .../java/hivemall/systemtest/model/HQ.java | 28 +++++++++---------- .../hivemall/systemtest/model/HQBase.java | 28 +++++++++---------- .../hivemall/systemtest/model/InsertHQ.java | 28 +++++++++---------- .../java/hivemall/systemtest/model/RawHQ.java | 28 +++++++++---------- .../hivemall/systemtest/model/TableHQ.java | 28 +++++++++---------- .../systemtest/model/TableListHQ.java | 28 +++++++++---------- .../model/UploadFileAsNewTableHQ.java | 28 +++++++++---------- .../systemtest/model/UploadFileHQ.java | 28 +++++++++---------- .../model/UploadFileToExistingHQ.java | 28 +++++++++---------- .../model/lazy/LazyMatchingResource.java | 28 +++++++++---------- .../runner/HiveSystemTestRunner.java | 28 +++++++++---------- .../runner/SystemTestCommonInfo.java | 28 +++++++++---------- .../systemtest/runner/SystemTestRunner.java | 28 +++++++++---------- .../systemtest/runner/SystemTestTeam.java | 28 +++++++++---------- .../systemtest/runner/TDSystemTestRunner.java | 28 +++++++++---------- .../java/hivemall/systemtest/utils/IO.java | 28 +++++++++---------- 23 files changed, 328 insertions(+), 295 deletions(-) diff --git a/systemtest/README.md b/systemtest/README.md index 9d1442aa..4fca0c3a 100644 --- a/systemtest/README.md +++ b/systemtest/README.md @@ -1,3 +1,21 @@ + ## Usage ### Initialization diff --git a/systemtest/pom.xml b/systemtest/pom.xml index e59d2ce6..e7345af8 100644 --- a/systemtest/pom.xml +++ b/systemtest/pom.xml @@ -1,4 +1,19 @@ - + diff --git a/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java b/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java index 99720f02..f7f372ff 100644 --- a/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java +++ b/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package com.klarna.hiverunner; diff --git a/systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java b/systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java index c6383e3b..b86c1cf8 100644 --- a/systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java +++ b/systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest; diff --git a/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java b/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java index e17a32ed..c2b00344 100644 --- a/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java +++ b/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.exception; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java index 40004b8b..e0047a65 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java index 4e9fe23d..c09ae247 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java index f847dfe2..05933a4c 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/HQBase.java b/systemtest/src/main/java/hivemall/systemtest/model/HQBase.java index 4212a5a0..00081002 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/HQBase.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQBase.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java index 30bf13b2..3d69f139 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java index 7671ca1c..1d6f0202 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java index 8c988766..2f9f3c94 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java index a4283bbb..146adbd5 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java index d88976e3..70915577 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java index 378521df..35b35c93 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java index fc7873ec..1f5c28d9 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model; diff --git a/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java index 3715a9e4..16f14eac 100644 --- a/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java +++ b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.model.lazy; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java index 09d242dc..6b418550 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.runner; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java index 5b55466b..60292fad 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.runner; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index e3d94120..77091f2d 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.runner; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index babbfbb7..86065e49 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.runner; diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index d646e7ee..6d6c85b7 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.runner; diff --git a/systemtest/src/main/java/hivemall/systemtest/utils/IO.java b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java index d572ed0e..04309451 100644 --- a/systemtest/src/main/java/hivemall/systemtest/utils/IO.java +++ b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java @@ -1,20 +1,20 @@ /* - * Hivemall: Hive scalable Machine Learning Library + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Copyright (C) 2016 Makoto YUI - * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * http://www.apache.org/licenses/LICENSE-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package hivemall.systemtest.utils; From 1e8093bfbbc8c48e6a2855bcde616ed2676ecf5a Mon Sep 17 00:00:00 2001 From: amaya Date: Thu, 17 Nov 2016 14:15:03 +0900 Subject: [PATCH 21/28] Mod README --- systemtest/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/systemtest/README.md b/systemtest/README.md index 4fca0c3a..28051654 100644 --- a/systemtest/README.md +++ b/systemtest/README.md @@ -157,7 +157,7 @@ public class QuickExample { public void test3() throws Exception { // test on HiveRunner once only // auto matching by files which name is `test3` in `case/` and `answer/` - team.set(HQ.autoMatchingByFileName("test3", ci)); // unordered test + team.set(HQ.autoMatchingByFileName("test3"), ci); // unordered test team.run(); // this call is required } @@ -165,7 +165,7 @@ public class QuickExample { public void test4() throws Exception { // test on HiveRunner once only predictor.expect(Throwable.class); // you can use systemtest w/ other rules - team.set(HQ.fromStatement("invalid queryyy")); // this query throws an exception + team.set(HQ.fromStatement("invalid queryyy"), "never used"); // this query throws an exception team.run(); // this call is required // thrown exception will be caught by `ExpectedException` rule } @@ -174,7 +174,7 @@ public class QuickExample { The above requires following files -* `systemtest/src/test/resources/hivemall/HogeTest/init/color.tsv` (`systemtest/src/test/resources/${path/to/package}/${className}/init/${fileName}`) +* `systemtest/src/test/resources/hivemall/QuickExample/init/color.tsv` (`systemtest/src/test/resources/${path/to/package}/${className}/init/${fileName}`) ```tsv blue 0 0 255 @@ -190,7 +190,7 @@ red 255 0 0 pink 255 192 203 ``` -* `systemtest/src/test/resources/hivemall/HogeTest/case/test3` (`systemtest/src/test/resources/${path/to/package}/${className}/case/${fileName}`) +* `systemtest/src/test/resources/hivemall/QuickExample/case/test3` (`systemtest/src/test/resources/${path/to/package}/${className}/case/${fileName}`) ```sql -- write your hive queries @@ -199,12 +199,12 @@ SELECT blue FROM color WHERE name = 'lavender';SELECT green FROM color WHERE nam SELECT name FROM color WHERE blue = 255 ``` -* `systemtest/src/test/resources/hivemall/HogeTest/answer/test3` (`systemtest/src/test/resources/${path/to/package}/${className}/answer/${fileName}`) +* `systemtest/src/test/resources/hivemall/QuickExample/answer/test3` (`systemtest/src/test/resources/${path/to/package}/${className}/answer/${fileName}`) tsv format is required ```tsv -230 +250 165 69 azure blue magenta ``` From f40ade095941c9218d46b29a913207bc54d86de6 Mon Sep 17 00:00:00 2001 From: amaya Date: Thu, 17 Nov 2016 14:16:14 +0900 Subject: [PATCH 22/28] Add exception --- .../java/hivemall/systemtest/runner/HiveSystemTestRunner.java | 1 + 1 file changed, 1 insertion(+) diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java index 6b418550..25a2125d 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java @@ -132,6 +132,7 @@ List uploadFileToExisting(@Nonnull final UploadFileToExistingHQ hq) thro hShell.insertInto(dbName, hq.tableName).addRowsFromTsv(hq.file).commit(); break; case MSGPACK: + throw new Exception("MessagePack is not supported in HiveSystemTestRunner"); case UNKNOWN: throw new Exception("Input csv or tsv"); } From 635a0a101e38496dd543372a4ec4a7452df13d56 Mon Sep 17 00:00:00 2001 From: amaya Date: Fri, 18 Nov 2016 01:57:31 +0900 Subject: [PATCH 23/28] Make dir name static --- .../systemtest/runner/SystemTestCommonInfo.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java index 60292fad..82b433f8 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java @@ -21,6 +21,10 @@ import javax.annotation.Nonnull; public class SystemTestCommonInfo { + private static final String CASE = "case"; + private static final String ANSWER = "answer"; + private static final String INIT = "init"; + @Nonnull public final String baseDir; @Nonnull @@ -34,9 +38,9 @@ public class SystemTestCommonInfo { public SystemTestCommonInfo(@Nonnull final Class clazz) { baseDir = clazz.getName().replace(".", "/"); - caseDir = baseDir + "/case/"; - answerDir = baseDir + "/answer/"; - initDir = baseDir + "/init/"; + caseDir = baseDir + "/" + CASE + "/"; + answerDir = baseDir + "/" + ANSWER + "/"; + initDir = baseDir + "/" + INIT + "/"; dbName = clazz.getName().replace(".", "_").toLowerCase(); } } From 06c88de8ae734ec97f25546930d37d88f515833f Mon Sep 17 00:00:00 2001 From: amaya Date: Fri, 18 Nov 2016 01:57:47 +0900 Subject: [PATCH 24/28] Mod assert methods --- .../java/hivemall/systemtest/runner/SystemTestRunner.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index 77091f2d..f16da90d 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -195,12 +195,10 @@ public void matching(@Nonnull final HQBase hq, @CheckForNull final String answer if (ordered) { // take order into consideration (like list) - Assert.assertThat(Arrays.asList(answer.split(IO.RD)), - Matchers.contains(result.toArray())); + Assert.assertThat(result, Matchers.contains(answer.split(IO.RD))); } else { // not take order into consideration (like multiset) - Assert.assertThat(Arrays.asList(answer.split(IO.RD)), - Matchers.containsInAnyOrder(result.toArray())); + Assert.assertThat(result, Matchers.containsInAnyOrder(answer.split(IO.RD))); } } From 4aa1cfc5867ab6b382bdd829cc552e57cad86f27 Mon Sep 17 00:00:00 2001 From: amaya Date: Fri, 18 Nov 2016 01:58:31 +0900 Subject: [PATCH 25/28] Fix process of tdprop --- .../systemtest/runner/TDSystemTestRunner.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index 6d6c85b7..87dd8354 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -85,16 +85,20 @@ protected void initRunner() { fileUploadCommitRetryLimit = Integer.valueOf(props.getProperty("fileUploadCommitRetryLimit")); } - final Properties TDPorps = System.getProperties(); + boolean fromPropertiesFile = false; for (Map.Entry e : props.entrySet()) { - if (e.getKey().toString().startsWith("td.client.")) { - TDPorps.setProperty(e.getKey().toString(), e.getValue().toString()); + final String key = e.getKey().toString(); + if (key.startsWith("td.client.")) { + fromPropertiesFile = true; + System.setProperty(key, e.getValue().toString()); } } - System.setProperties(TDPorps); - client = System.getProperties().size() == TDPorps.size() ? TDClient.newClient() // use $HOME/.td/td.conf - : TDClient.newBuilder(false).build(); // use *.properties + if (fromPropertiesFile) { + client = TDClient.newBuilder(false).build(); // use *.properties + } else { + client = TDClient.newClient(); // use $HOME/.td/td.conf + } } @Override From bdf38c2acba4fe17323e61f719a318be8f7fc8e8 Mon Sep 17 00:00:00 2001 From: amaya Date: Fri, 18 Nov 2016 03:23:46 +0900 Subject: [PATCH 26/28] Mod README --- systemtest/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/systemtest/README.md b/systemtest/README.md index 28051654..2b1167e2 100644 --- a/systemtest/README.md +++ b/systemtest/README.md @@ -195,8 +195,9 @@ pink 255 192 203 ```sql -- write your hive queries -- comments like this and multiple queries in one row are allowed -SELECT blue FROM color WHERE name = 'lavender';SELECT green FROM color WHERE name LIKE 'orange%' -SELECT name FROM color WHERE blue = 255 +SELECT blue FROM color WHERE name = 'lavender'; +SELECT green FROM color WHERE name LIKE 'orange%'; +SELECT name FROM color WHERE blue = 255; ``` * `systemtest/src/test/resources/hivemall/QuickExample/answer/test3` (`systemtest/src/test/resources/${path/to/package}/${className}/answer/${fileName}`) @@ -205,6 +206,6 @@ tsv format is required ```tsv 250 -165 69 -azure blue magenta +165 69 +azure blue magenta ``` From 25db0c030523111564d89982f0a57921fd2ce093 Mon Sep 17 00:00:00 2001 From: amaya Date: Fri, 18 Nov 2016 04:22:51 +0900 Subject: [PATCH 27/28] Refine access modifiers/calls --- .../runner/HiveSystemTestRunner.java | 4 +- .../systemtest/runner/SystemTestRunner.java | 40 ++++++++++--------- .../systemtest/runner/SystemTestTeam.java | 8 +--- .../systemtest/runner/TDSystemTestRunner.java | 24 +++++------ 4 files changed, 36 insertions(+), 40 deletions(-) diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java index 25a2125d..db1edc7b 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java @@ -101,7 +101,7 @@ void initRunner() { } @Override - protected void finRunner() { + void finRunner() { if (container != null) { container.tearDown(); } @@ -111,7 +111,7 @@ protected void finRunner() { } @Override - protected List exec(@Nonnull final RawHQ hq) { + public List exec(@Nonnull final RawHQ hq) { logger.info("executing: `" + hq.query + "`"); return hShell.executeQuery(hq.query); diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java index f16da90d..e1421745 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -45,7 +45,6 @@ import java.io.FileInputStream; import java.io.InputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -55,9 +54,9 @@ public abstract class SystemTestRunner extends ExternalResource { static final Logger logger = LoggerFactory.getLogger(SystemTestRunner.class); @Nonnull - final List classInitHqs; + private final List classInitHqs; @Nonnull - final Set immutableTables; + private final Set immutableTables; @Nonnull final String dbName; @Nonnull @@ -98,7 +97,7 @@ protected void before() throws Exception { @Override protected void after() { try { - resetDB(); // clean up database + cleanDB(); // clean up database } catch (Exception ex) { throw new QueryExecutionException("Failed to clean up temporary database. " + ex.getMessage()); @@ -111,16 +110,16 @@ protected void after() { abstract void finRunner(); - public void initBy(@Nonnull final HQBase hq) { + protected void initBy(@Nonnull final HQBase hq) { classInitHqs.add(hq); } - public void initBy(@Nonnull final List hqs) { + protected void initBy(@Nonnull final List hqs) { classInitHqs.addAll(hqs); } // fix to temporary database and user-defined init (should be called per Test class) - void prepareDB() throws Exception { + private void prepareDB() throws Exception { createDB(dbName); use(dbName); for (HQBase q : classInitHqs) { @@ -136,15 +135,21 @@ void prepareDB() throws Exception { } // drop temporary database (should be called per Test class) - void resetDB() throws Exception { + private void cleanDB() throws Exception { dropDB(dbName); } - public final boolean isImmutableTable(final String tableName) { - return immutableTables.contains(tableName); + // drop temporary tables (should be called per Test method) + void resetDB() throws Exception { + final List tables = tableList(); + for (String t : tables) { + if (!immutableTables.contains(t)) { + dropTable(HQ.dropTable(t)); + } + } } - // execute HQBase + // >execute HQBase public List exec(@Nonnull final HQBase hq) throws Exception { if (hq instanceof RawHQ) { return exec((RawHQ) hq); @@ -157,10 +162,10 @@ public List exec(@Nonnull final HQBase hq) throws Exception { } } - //// execute RawHQ + // >>execute RawHQ abstract protected List exec(@Nonnull final RawHQ hq) throws Exception; - //// execute TableHQ + // >>execute TableHQ List exec(@Nonnull final TableHQ hq) throws Exception { if (hq instanceof CreateTableHQ) { return createTable((CreateTableHQ) hq); @@ -175,7 +180,7 @@ List exec(@Nonnull final TableHQ hq) throws Exception { } } - ////// execute UploadFileHQ + // >>>execute UploadFileHQ List exec(@Nonnull final UploadFileHQ hq) throws Exception { if (hq instanceof UploadFileAsNewTableHQ) { return uploadFileAsNewTable((UploadFileAsNewTableHQ) hq); @@ -187,8 +192,8 @@ List exec(@Nonnull final UploadFileHQ hq) throws Exception { } // matching HQBase - public void matching(@Nonnull final HQBase hq, @CheckForNull final String answer, - final boolean ordered) throws Exception { + void matching(@Nonnull final HQBase hq, @CheckForNull final String answer, final boolean ordered) + throws Exception { Preconditions.checkNotNull(answer); List result = exec(hq); @@ -203,8 +208,7 @@ public void matching(@Nonnull final HQBase hq, @CheckForNull final String answer } // matching HQBase (ordered == false) - public void matching(@Nonnull final HQBase hq, @CheckForNull final String answer) - throws Exception { + void matching(@Nonnull final HQBase hq, @CheckForNull final String answer) throws Exception { matching(hq, answer, false); } diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java index 86065e49..fcd2fcb3 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -19,7 +19,6 @@ package hivemall.systemtest.runner; import hivemall.systemtest.exception.QueryExecutionException; -import hivemall.systemtest.model.HQ; import hivemall.systemtest.model.HQBase; import hivemall.systemtest.model.RawHQ; import hivemall.systemtest.model.lazy.LazyMatchingResource; @@ -66,12 +65,7 @@ protected void after() { for (SystemTestRunner runner : reachGoal) { try { - final List tables = runner.exec(HQ.tableList()); - for (String t : tables) { - if (!runner.isImmutableTable(t)) { - runner.exec(HQ.dropTable(t)); - } - } + runner.resetDB(); } catch (Exception ex) { throw new QueryExecutionException("Failed to resetPerMethod database. " + ex.getMessage()); diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java index 87dd8354..b2f82903 100644 --- a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -70,7 +70,7 @@ public TDSystemTestRunner(final SystemTestCommonInfo ci) { } @Override - protected void initRunner() { + void initRunner() { // optional if (props.containsKey("execFinishRetryLimit")) { execFinishRetryLimit = Integer.valueOf(props.getProperty("execFinishRetryLimit")); @@ -102,14 +102,14 @@ protected void initRunner() { } @Override - protected void finRunner() { + void finRunner() { if (client != null) { client.close(); } } @Override - protected List exec(@Nonnull final RawHQ hq) throws Exception { + public List exec(@Nonnull final RawHQ hq) throws Exception { logger.info("executing: `" + hq.query + "`"); final TDJobRequest req = TDJobRequest.newHiveQuery(dbName, hq.query); @@ -157,7 +157,7 @@ public List apply(InputStream input) { } @Override - protected List createDB(@Nonnull final String dbName) throws Exception { + List createDB(@Nonnull final String dbName) throws Exception { logger.info("executing: create database if not exists " + dbName); client.createDatabaseIfNotExists(dbName); @@ -165,7 +165,7 @@ protected List createDB(@Nonnull final String dbName) throws Exception { } @Override - protected List dropDB(@Nonnull final String dbName) throws Exception { + List dropDB(@Nonnull final String dbName) throws Exception { logger.info("executing: drop database if exists " + dbName); client.deleteDatabaseIfExists(dbName); @@ -173,13 +173,13 @@ protected List dropDB(@Nonnull final String dbName) throws Exception { } @Override - protected List use(@Nonnull final String dbName) throws Exception { + List use(@Nonnull final String dbName) throws Exception { return Collections.singletonList("No need to execute `USE` statement on TD, so skipped `USE " + dbName + "`"); } @Override - protected List tableList() throws Exception { + List tableList() throws Exception { logger.info("executing: show tables on " + dbName); final List tables = client.listTables(dbName); @@ -191,7 +191,7 @@ protected List tableList() throws Exception { } @Override - protected List createTable(@Nonnull final CreateTableHQ hq) throws Exception { + List createTable(@Nonnull final CreateTableHQ hq) throws Exception { logger.info("executing: create table " + hq.tableName + " if not exists on " + dbName); final List columns = new ArrayList(); @@ -204,7 +204,7 @@ protected List createTable(@Nonnull final CreateTableHQ hq) throws Excep } @Override - protected List dropTable(@Nonnull final DropTableHQ hq) throws Exception { + List dropTable(@Nonnull final DropTableHQ hq) throws Exception { logger.info("executing: drop table " + hq.tableName + " if exists on " + dbName); client.deleteTableIfExists(dbName, hq.tableName); @@ -212,8 +212,7 @@ protected List dropTable(@Nonnull final DropTableHQ hq) throws Exception } @Override - protected List uploadFileAsNewTable(@Nonnull final UploadFileAsNewTableHQ hq) - throws Exception { + List uploadFileAsNewTable(@Nonnull final UploadFileAsNewTableHQ hq) throws Exception { logger.info("executing: create " + hq.tableName + " based on " + hq.file.getPath() + " if not exists on " + dbName); @@ -292,8 +291,7 @@ protected List uploadFileAsNewTable(@Nonnull final UploadFileAsNewTableH } @Override - protected List uploadFileToExisting(@Nonnull final UploadFileToExistingHQ hq) - throws Exception { + List uploadFileToExisting(@Nonnull final UploadFileToExistingHQ hq) throws Exception { logger.info("executing: insert " + hq.file.getPath() + " into " + hq.tableName + " on " + dbName); From f2ae1f7ca4820bc6903bbdae2f8c7b8156745d03 Mon Sep 17 00:00:00 2001 From: amaya Date: Fri, 18 Nov 2016 04:27:59 +0900 Subject: [PATCH 28/28] Update license header --- systemtest/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/systemtest/pom.xml b/systemtest/pom.xml index e7345af8..0debee03 100644 --- a/systemtest/pom.xml +++ b/systemtest/pom.xml @@ -6,7 +6,9 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY