diff --git a/common/core/src/main/java/zingg/common/core/hash/IdentityLong.java b/common/core/src/main/java/zingg/common/core/hash/IdentityLong.java new file mode 100644 index 000000000..cbc24a54d --- /dev/null +++ b/common/core/src/main/java/zingg/common/core/hash/IdentityLong.java @@ -0,0 +1,16 @@ +package zingg.common.core.hash; + + +public class IdentityLong extends BaseHash{ + + private static final long serialVersionUID = 1L; + + public IdentityLong() { + setName("identityLong"); + } + + public Long call(Long field) { + return field; + } + +} diff --git a/common/core/src/main/java/zingg/common/core/hash/LessThanZeroLong.java b/common/core/src/main/java/zingg/common/core/hash/LessThanZeroLong.java new file mode 100644 index 000000000..52fd64c38 --- /dev/null +++ b/common/core/src/main/java/zingg/common/core/hash/LessThanZeroLong.java @@ -0,0 +1,18 @@ +package zingg.common.core.hash; + +public class LessThanZeroLong extends BaseHash{ + private static final long serialVersionUID = 1L; + + public LessThanZeroLong() { + setName("lessThanZeroLong"); + } + + public Boolean call(Long field) { + Boolean r = false; + if (field != null) { + r = field < 0 ? true : false; + } + return r; + } + +} diff --git a/common/core/src/main/java/zingg/common/core/hash/RangeLong.java b/common/core/src/main/java/zingg/common/core/hash/RangeLong.java new file mode 100644 index 000000000..f0c92328f --- /dev/null +++ b/common/core/src/main/java/zingg/common/core/hash/RangeLong.java @@ -0,0 +1,33 @@ +package zingg.common.core.hash; + +public class RangeLong extends BaseHash{ + private static final long serialVersionUID = 1L; + private long lowerLimit; + private long upperLimit; + + public RangeLong(long lower, long upper) { + setName("rangeBetween" + lower + "And" + upper + "Long"); + this.lowerLimit = lower; + this.upperLimit = upper; + } + + + public Long call(Long field) { + long withinRange = 0; + if (field != null && field >= lowerLimit && field < upperLimit) { + withinRange = 1; + } + return withinRange; + } + + + public long getLowerLimit() { + return lowerLimit; + } + + + public long getUpperLimit() { + return upperLimit; + } + +} diff --git a/common/core/src/main/java/zingg/common/core/hash/TrimLastDigitsLong.java b/common/core/src/main/java/zingg/common/core/hash/TrimLastDigitsLong.java new file mode 100644 index 000000000..9e15c5687 --- /dev/null +++ b/common/core/src/main/java/zingg/common/core/hash/TrimLastDigitsLong.java @@ -0,0 +1,30 @@ +package zingg.common.core.hash; + +/** + * Base class for hash functions related to trimming of longs + * + */ +public class TrimLastDigitsLong extends BaseHash{ + private static final long serialVersionUID = 1L; + private int numDigits; + static final int[] POWERS_OF_10 = {1, 10, 100, 1000, 10000, 100000}; + public TrimLastDigitsLong(int count) { + setName("trimLast" + count + "DigitsLong"); + this.numDigits = count; + } + + public Long call(Long field) { + Long r = null; + if (field == null) { + r = field; + } else { + r = field / POWERS_OF_10[numDigits]; + } + return r; + } + + public int getNumDigits() { + return numDigits; + } + +} diff --git a/common/core/src/main/resources/hashFunctions.json b/common/core/src/main/resources/hashFunctions.json index 9043dad38..bc536142b 100644 --- a/common/core/src/main/resources/hashFunctions.json +++ b/common/core/src/main/resources/hashFunctions.json @@ -39,6 +39,9 @@ { "name":"identityInteger" }, + { + "name":"identityLong" + }, { "name":"truncateDoubleTo1Places" }, @@ -54,6 +57,9 @@ { "name":"lessThanZeroInt" }, + { + "name":"lessThanZeroLong" + }, { "name":"trimLast1DigitsDbl" }, @@ -72,6 +78,15 @@ { "name":"trimLast3DigitsInt" }, + { + "name":"trimLast1DigitsLong" + }, + { + "name":"trimLast2DigitsLong" + }, + { + "name":"trimLast3DigitsLong" + }, { "name":"rangeBetween0And10Dbl" }, @@ -96,6 +111,18 @@ { "name":"rangeBetween1000And10000Int" }, + { + "name":"rangeBetween0And10Long" + }, + { + "name":"rangeBetween10And100Long" + }, + { + "name":"rangeBetween100And1000Long" + }, + { + "name":"rangeBetween1000And10000Long" + }, { "name":"round" } diff --git a/spark/core/src/main/java/zingg/spark/core/hash/SparkHashFunctionRegistry.java b/spark/core/src/main/java/zingg/spark/core/hash/SparkHashFunctionRegistry.java index 30d3a69cc..ed33c9214 100644 --- a/spark/core/src/main/java/zingg/spark/core/hash/SparkHashFunctionRegistry.java +++ b/spark/core/src/main/java/zingg/spark/core/hash/SparkHashFunctionRegistry.java @@ -40,6 +40,8 @@ public SparkHashFunctionRegistry() { init(new SparkFirst3CharsBox()); init(new SparkIdentityInteger()); + + init(new SparkIdentityLong()); init(new SparkTruncateDouble(1)); init(new SparkTruncateDouble(2)); @@ -49,6 +51,8 @@ public SparkHashFunctionRegistry() { init(new SparkLessThanZeroInt()); + init(new SparkLessThanZeroLong()); + init(new SparkTrimLastDigitsDbl(1)); init(new SparkTrimLastDigitsDbl(2)); init(new SparkTrimLastDigitsDbl(3)); @@ -57,6 +61,10 @@ public SparkHashFunctionRegistry() { init(new SparkTrimLastDigitsInt(2)); init(new SparkTrimLastDigitsInt(3)); + init(new SparkTrimLastDigitsLong(1)); + init(new SparkTrimLastDigitsLong(2)); + init(new SparkTrimLastDigitsLong(3)); + init(new SparkRangeDbl(0,10)); init(new SparkRangeDbl(10,100)); init(new SparkRangeDbl(100,1000)); @@ -67,6 +75,11 @@ public SparkHashFunctionRegistry() { init(new SparkRangeInt(100,1000)); init(new SparkRangeInt(1000,10000)); + init(new SparkRangeLong(0,10)); + init(new SparkRangeLong(10,100)); + init(new SparkRangeLong(100,1000)); + init(new SparkRangeLong(1000,10000)); + init(new SparkRound()); } diff --git a/spark/core/src/main/java/zingg/spark/core/hash/SparkIdentityLong.java b/spark/core/src/main/java/zingg/spark/core/hash/SparkIdentityLong.java new file mode 100644 index 000000000..8770afcb0 --- /dev/null +++ b/spark/core/src/main/java/zingg/spark/core/hash/SparkIdentityLong.java @@ -0,0 +1,20 @@ +package zingg.spark.core.hash; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.spark.sql.types.DataTypes; + +import zingg.common.core.hash.IdentityLong; + +public class SparkIdentityLong extends SparkHashFunction{ + + private static final long serialVersionUID = 1L; + public static final Log LOG = LogFactory.getLog(SparkIdentityLong.class); + + public SparkIdentityLong() { + setBaseHash(new IdentityLong()); + setDataType(DataTypes.LongType); + setReturnType(DataTypes.LongType); + } + +} diff --git a/spark/core/src/main/java/zingg/spark/core/hash/SparkLessThanZeroLong.java b/spark/core/src/main/java/zingg/spark/core/hash/SparkLessThanZeroLong.java new file mode 100644 index 000000000..6d72d300b --- /dev/null +++ b/spark/core/src/main/java/zingg/spark/core/hash/SparkLessThanZeroLong.java @@ -0,0 +1,20 @@ +package zingg.spark.core.hash; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.spark.sql.types.DataTypes; + +import zingg.common.core.hash.LessThanZeroLong; + +public class SparkLessThanZeroLong extends SparkHashFunction{ + + private static final long serialVersionUID = 1L; + public static final Log LOG = LogFactory.getLog(SparkLessThanZeroLong.class); + + public SparkLessThanZeroLong() { + setBaseHash(new LessThanZeroLong()); + setDataType(DataTypes.LongType); + setReturnType(DataTypes.BooleanType); + } + +} diff --git a/spark/core/src/main/java/zingg/spark/core/hash/SparkRangeLong.java b/spark/core/src/main/java/zingg/spark/core/hash/SparkRangeLong.java new file mode 100644 index 000000000..e3649a371 --- /dev/null +++ b/spark/core/src/main/java/zingg/spark/core/hash/SparkRangeLong.java @@ -0,0 +1,20 @@ +package zingg.spark.core.hash; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.spark.sql.types.DataTypes; + +import zingg.common.core.hash.RangeLong; + +public class SparkRangeLong extends SparkHashFunction{ + + private static final long serialVersionUID = 1L; + public static final Log LOG = LogFactory.getLog(SparkRangeLong.class); + + public SparkRangeLong(long lower, long upper) { + setBaseHash(new RangeLong(lower ,upper)); + setDataType(DataTypes.LongType); + setReturnType(DataTypes.LongType); + } + +} diff --git a/spark/core/src/main/java/zingg/spark/core/hash/SparkTrimLastDigitsLong.java b/spark/core/src/main/java/zingg/spark/core/hash/SparkTrimLastDigitsLong.java new file mode 100644 index 000000000..5f63729e1 --- /dev/null +++ b/spark/core/src/main/java/zingg/spark/core/hash/SparkTrimLastDigitsLong.java @@ -0,0 +1,25 @@ +package zingg.spark.core.hash; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.spark.sql.types.DataTypes; + +import zingg.common.core.hash.TrimLastDigitsLong; + +/** + * Spark specific trim function for Long + * + * + */ +public class SparkTrimLastDigitsLong extends SparkHashFunction{ + + private static final long serialVersionUID = 1L; + public static final Log LOG = LogFactory.getLog(SparkTrimLastDigitsLong.class); + + public SparkTrimLastDigitsLong(int count){ + setBaseHash(new TrimLastDigitsLong(count)); + setDataType(DataTypes.LongType); + setReturnType(DataTypes.LongType); + } + +}