diff --git ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java index 2e369ec..958233b 100644 --- ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java +++ ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java @@ -77,15 +77,6 @@ {"DTIScalarArithmeticDTIColumnNoConvert", "Subtract", "interval_year_month", "interval_year_month", "-"}, {"DTIColumnArithmeticDTIColumnNoConvert", "Subtract", "interval_year_month", "interval_year_month", "-"}, - // Arithmetic on two TimestampColumnVector base classes. - {"TimestampArithmeticTimestampBase", "Add", "Col", "Column"}, - {"TimestampArithmeticTimestampBase", "Add", "Scalar", "Column"}, - {"TimestampArithmeticTimestampBase", "Add", "Col", "Scalar"}, - - {"TimestampArithmeticTimestampBase", "Subtract", "Col", "Column"}, - {"TimestampArithmeticTimestampBase", "Subtract", "Scalar", "Column"}, - {"TimestampArithmeticTimestampBase", "Subtract", "Col", "Scalar"}, - // Arithmetic on two type interval_day_time (TimestampColumnVector storing nanosecond interval // in 2 longs) produces a interval_day_time. {"TimestampArithmeticTimestamp", "Add", "interval_day_time", "Col", "interval_day_time", "Scalar"}, @@ -111,30 +102,13 @@ {"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Col", "interval_day_time", "Column"}, // A type timestamp (TimestampColumnVector) minus a type timestamp produces a - // type interval_day_time (TimestampColumnVector storing nanosecond interval in 2 longs). + // type interval_day_time (IntervalDayTimeColumnVector storing nanosecond interval in 2 primitives). {"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Col", "timestamp", "Scalar"}, {"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Scalar", "timestamp", "Column"}, {"TimestampArithmeticTimestamp", "Subtract", "timestamp", "Col", "timestamp", "Column"}, - // Arithmetic on a TimestampColumnVector and date base classes. - {"DateArithmeticTimestampBase", "Add", "Col", "Column"}, - {"DateArithmeticTimestampBase", "Add", "Scalar", "Column"}, - {"DateArithmeticTimestampBase", "Add", "Col", "Scalar"}, - - {"DateArithmeticTimestampBase", "Subtract", "Col", "Column"}, - {"DateArithmeticTimestampBase", "Subtract", "Scalar", "Column"}, - {"DateArithmeticTimestampBase", "Subtract", "Col", "Scalar"}, - - {"TimestampArithmeticDateBase", "Add", "Col", "Column"}, - {"TimestampArithmeticDateBase", "Add", "Scalar", "Column"}, - {"TimestampArithmeticDateBase", "Add", "Col", "Scalar"}, - - {"TimestampArithmeticDateBase", "Subtract", "Col", "Column"}, - {"TimestampArithmeticDateBase", "Subtract", "Scalar", "Column"}, - {"TimestampArithmeticDateBase", "Subtract", "Col", "Scalar"}, - - // Arithmetic with a type date (LongColumnVector storing epoch days) and type interval_day_time (TimestampColumnVector storing - // nanosecond interval in 2 longs) produces a type timestamp (TimestampColumnVector). + // Arithmetic with a type date (LongColumnVector storing epoch days) and type interval_day_time (IntervalDayTimeColumnVector storing + // nanosecond interval in 2 primitives) produces a type timestamp (TimestampColumnVector). {"DateArithmeticTimestamp", "Add", "date", "Col", "interval_day_time", "Column"}, {"DateArithmeticTimestamp", "Add", "date", "Scalar", "interval_day_time", "Column"}, {"DateArithmeticTimestamp", "Add", "date", "Col", "interval_day_time", "Scalar"}, @@ -147,7 +121,8 @@ {"TimestampArithmeticDate", "Add", "interval_day_time", "Scalar", "date", "Column"}, {"TimestampArithmeticDate", "Add", "interval_day_time", "Col", "date", "Scalar"}, - // Subtraction with a type date (LongColumnVector storing epoch days) and type timestamp produces a type timestamp (TimestampColumnVector). + // Subtraction with a type date (LongColumnVector storing days) and type timestamp produces a + // type interval_day_time (IntervalDayTimeColumnVector). {"DateArithmeticTimestamp", "Subtract", "date", "Col", "timestamp", "Column"}, {"DateArithmeticTimestamp", "Subtract", "date", "Scalar", "timestamp", "Column"}, {"DateArithmeticTimestamp", "Subtract", "date", "Col", "timestamp", "Scalar"}, @@ -318,70 +293,48 @@ {"ScalarCompareColumn", "Greater", "double", "long", ">"}, {"ScalarCompareColumn", "GreaterEqual", "double", "long", ">="}, - // Base compare timestamp to timestamp used by Timestamp and IntervalDayTime. - {"TimestampCompareTimestampBase", "Equal", "==", "Col", "Column"}, - {"TimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Column"}, - {"TimestampCompareTimestampBase", "Less", "<", "Col", "Column"}, - {"TimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Column"}, - {"TimestampCompareTimestampBase", "Greater", ">", "Col", "Column"}, - {"TimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Column"}, - - {"TimestampCompareTimestampBase", "Equal", "==", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "Less", "<", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "Greater", ">", "Col", "Scalar"}, - {"TimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Scalar"}, - - {"TimestampCompareTimestampBase", "Equal", "==", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "NotEqual", "!=", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "Less", "<", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "LessEqual", "<=", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "Greater", ">", "Scalar", "Column"}, - {"TimestampCompareTimestampBase", "GreaterEqual", ">=", "Scalar", "Column"}, - // Compare timestamp to timestamp. - {"TimestampCompareTimestamp", "Equal", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "Less", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "Greater", "timestamp", "Col", "Column"}, - {"TimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Column"}, - - {"TimestampCompareTimestamp", "Equal", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "Less", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "Greater", "timestamp", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Scalar"}, - - {"TimestampCompareTimestamp", "Equal", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "NotEqual", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "Less", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "LessEqual", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "Greater", "timestamp", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "GreaterEqual", "timestamp", "Scalar", "Column"}, - - {"TimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Column"}, - {"TimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Column"}, - - {"TimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Scalar"}, - {"TimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Scalar"}, - - {"TimestampCompareTimestamp", "Equal", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "NotEqual", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "Less", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "LessEqual", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "Greater", "interval_day_time", "Scalar", "Column"}, - {"TimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Column"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Column"}, + + {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Scalar"}, + + {"TimestampCompareTimestamp", "Equal", "==", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Less", "<", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Greater", ">", "timestamp", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Scalar", "Column"}, + + {"TimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Column"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Column"}, + + {"TimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Scalar"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Scalar"}, + + {"TimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Less", "<", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Scalar", "Column"}, + {"TimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Scalar", "Column"}, // Compare timestamp to integer seconds or double seconds with fractional nanoseonds. {"TimestampCompareLongDouble", "Equal", "long", "==", "Col", "Column"}, @@ -515,71 +468,49 @@ {"FilterScalarCompareColumn", "GreaterEqual", "long", "long", ">="}, {"FilterScalarCompareColumn", "GreaterEqual", "double", "long", ">="}, - // Base filter timestamp against timestamp used by Timestamp and IntervalDayTime. - {"FilterTimestampCompareTimestampBase", "Equal", "==", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "Less", "<", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "Greater", ">", "Col", "Column"}, - {"FilterTimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Column"}, - - {"FilterTimestampCompareTimestampBase", "Equal", "==", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "NotEqual", "!=", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "Less", "<", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "LessEqual", "<=", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "Greater", ">", "Col", "Scalar"}, - {"FilterTimestampCompareTimestampBase", "GreaterEqual", ">=", "Col", "Scalar"}, - - {"FilterTimestampCompareTimestampBase", "Equal", "==", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "NotEqual", "!=", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "Less", "<", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "LessEqual", "<=", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "Greater", ">", "Scalar", "Column"}, - {"FilterTimestampCompareTimestampBase", "GreaterEqual", ">=", "Scalar", "Column"}, - // Filter timestamp against timestamp, or interval day time against interval day time. - {"FilterTimestampCompareTimestamp", "Equal", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "Less", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "Greater", "timestamp", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Column"}, - - {"FilterTimestampCompareTimestamp", "Equal", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "Less", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "Greater", "timestamp", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "timestamp", "Col", "Scalar"}, - - {"FilterTimestampCompareTimestamp", "Equal", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "Less", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "Greater", "timestamp", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "timestamp", "Scalar", "Column"}, - - {"FilterTimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Column"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Column"}, - - {"FilterTimestampCompareTimestamp", "Equal", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "Less", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "Greater", "interval_day_time", "Col", "Scalar"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Col", "Scalar"}, - - {"FilterTimestampCompareTimestamp", "Equal", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "NotEqual", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "Less", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "LessEqual", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "Greater", "interval_day_time", "Scalar", "Column"}, - {"FilterTimestampCompareTimestamp", "GreaterEqual", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Column"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "timestamp", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Col", "Scalar"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "timestamp", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "timestamp", "Scalar", "Column"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Column"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Column"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Col", "Scalar"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Col", "Scalar"}, + + {"FilterTimestampCompareTimestamp", "Equal", "==", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "NotEqual", "!=", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Less", "<", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "LessEqual", "<=", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "Greater", ">", "interval_day_time", "Scalar", "Column"}, + {"FilterTimestampCompareTimestamp", "GreaterEqual", ">=", "interval_day_time", "Scalar", "Column"}, // Filter timestamp against long (seconds) or double (seconds with fractional // nanoseconds). @@ -1202,9 +1133,6 @@ private void generate() throws Exception { } else if (tdesc[0].equals("ScalarCompareColumn")) { generateScalarCompareColumn(tdesc); - } else if (tdesc[0].equals("TimestampCompareTimestampBase")) { - generateTimestampCompareTimestampBase(tdesc); - } else if (tdesc[0].equals("TimestampCompareTimestamp")) { generateTimestampCompareTimestamp(tdesc); @@ -1219,9 +1147,6 @@ private void generate() throws Exception { } else if (tdesc[0].equals("FilterScalarCompareColumn")) { generateFilterScalarCompareColumn(tdesc); - } else if (tdesc[0].equals("FilterTimestampCompareTimestampBase")) { - generateFilterTimestampCompareTimestampBase(tdesc); - } else if (tdesc[0].equals("FilterTimestampCompareTimestamp")) { generateFilterTimestampCompareTimestamp(tdesc); @@ -1338,21 +1263,12 @@ private void generate() throws Exception { } else if (tdesc[0].equals("IntervalYearMonthArithmeticTimestamp")) { generateDateTimeArithmeticIntervalYearMonth(tdesc); - } else if (tdesc[0].equals("TimestampArithmeticTimestampBase")) { - generateTimestampArithmeticTimestampBase(tdesc); - } else if (tdesc[0].equals("TimestampArithmeticTimestamp")) { generateTimestampArithmeticTimestamp(tdesc); - } else if (tdesc[0].equals("DateArithmeticTimestampBase")) { - generateDateArithmeticTimestampBase(tdesc); - } else if (tdesc[0].equals("DateArithmeticTimestamp")) { generateDateArithmeticTimestamp(tdesc); - } else if (tdesc[0].equals("TimestampArithmeticDateBase")) { - generateTimestampArithmeticDateBase(tdesc); - } else if (tdesc[0].equals("TimestampArithmeticDate")) { generateTimestampArithmeticDate(tdesc); @@ -2182,35 +2098,28 @@ private void generateColumnCompareOperatorColumn(String[] tdesc, boolean filter, // // ----------------------------------------------------------------------------------------------- - private void generateFilterTimestampCompareTimestampBase(String[] tdesc) throws Exception { + private void generateFilterTimestampCompareTimestamp(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operatorSymbol = tdesc[2]; - String className = "FilterTimestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base"; + String operandType = tdesc[3]; + String camelOperandType = getCamelCaseType(operandType); + String className = "Filter" + camelOperandType + tdesc[4] + operatorName + camelOperandType + tdesc[5]; + String baseClassName = "FilterTimestamp" + tdesc[4] + operatorName + "Timestamp" + tdesc[5] + "Base"; //Read the template into a string; - String fileName = "FilterTimestamp" + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "CompareTimestamp" + - tdesc[4] + "Base"; + String fileName = "FilterTimestamp" + (tdesc[4].equals("Col") ? "Column" : tdesc[4]) + "CompareTimestamp" + + tdesc[5]; File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", operatorSymbol); - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); - } + templateString = templateString.replaceAll("", operandType); + templateString = templateString.replaceAll("", camelOperandType); + templateString = templateString.replaceAll("", getTimestampHiveType(operandType)); + + String inputColumnVectorType = this.getColumnVectorType(operandType); + templateString = templateString.replaceAll("", inputColumnVectorType); - private void generateFilterTimestampCompareTimestamp(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String operandType = tdesc[2]; - String camelCaseOperandType = getCamelCaseType(operandType); - String className = "Filter" + camelCaseOperandType + tdesc[3] + operatorName + camelCaseOperandType + tdesc[4]; - String baseClassName = "FilterTimestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base"; - //Read the template into a string; - String fileName = "Filter" + camelCaseOperandType + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "Compare" + camelCaseOperandType + - tdesc[4]; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", baseClassName); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } @@ -2287,9 +2196,9 @@ private void generateFilterLongDoubleCompareTimestamp(String[] tdesc) throws Exc private String timestampLongDoubleMethod(String operandType) { if (operandType.equals("long")) { - return "getTimestampSeconds"; + return "getTimestampAsLong"; } else if (operandType.equals("double")) { - return "getTimestampSecondsWithFractionalNanos"; + return "getTimestampAsDouble"; } else { return "unknown"; } @@ -2314,35 +2223,26 @@ private String timestampLongDoubleMethod(String operandType) { // // ----------------------------------------------------------------------------------------------- - private void generateTimestampCompareTimestampBase(String[] tdesc) throws Exception { + + private void generateTimestampCompareTimestamp(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operatorSymbol = tdesc[2]; - String className = "Timestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base"; + String operandType = tdesc[3]; + String camelOperandType = getCamelCaseType(operandType); + String className = camelOperandType + tdesc[4] + operatorName + camelOperandType + tdesc[5]; //Read the template into a string; - String fileName = "Timestamp" + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "CompareTimestamp" + - tdesc[4] + "Base"; + String fileName = "Timestamp" + (tdesc[4].equals("Col") ? "Column" : tdesc[4]) + "CompareTimestamp" + + (tdesc[5].equals("Col") ? "Column" : tdesc[5]); File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", operatorSymbol); - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); - } + templateString = templateString.replaceAll("", operandType); + templateString = templateString.replaceAll("", camelOperandType); + templateString = templateString.replaceAll("", getTimestampHiveType(operandType)); + templateString = templateString.replaceAll("", getColumnVectorType(operandType)); - private void generateTimestampCompareTimestamp(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String operandType = tdesc[2]; - String camelCaseOperandType = getCamelCaseType(operandType); - String className = camelCaseOperandType + tdesc[3] + operatorName + camelCaseOperandType + tdesc[4]; - String baseClassName = "Timestamp" + tdesc[3] + operatorName + "Timestamp" + tdesc[4] + "Base"; - //Read the template into a string; - String fileName = camelCaseOperandType + (tdesc[3].equals("Col") ? "Column" : tdesc[3]) + "Compare" + camelCaseOperandType + - tdesc[4]; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", baseClassName); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); } @@ -2851,6 +2751,7 @@ private void generateDateTimeArithmeticIntervalYearMonth(String[] tdesc) throws String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", operatorSymbol); + templateString = templateString.replaceAll("", operatorName.toLowerCase()); writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, className, templateString); @@ -2924,30 +2825,6 @@ private String replaceTimestampScalar(String templateString, int argNum, String return templateString; } - // TimestampColumnArithmeticTimestampColumnBase.txt - // TimestampScalarArithmeticTimestampColumnBase.txt - // TimestampColumnArithmeticTimestampScalarBase.txt - // - private void generateTimestampArithmeticTimestampBase(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String colOrScalar1 = tdesc[2]; - String colOrScalar2 = tdesc[3]; - - String baseClassName = "Timestamp" + colOrScalar1 + operatorName + - "Timestamp" + colOrScalar2 + "Base"; - - //Read the template into a string; - String fileName = "Timestamp" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + - "Timestamp" + colOrScalar2 + "Base"; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", baseClassName); - templateString = templateString.replaceAll("", operatorName.toLowerCase()); - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - baseClassName, templateString); - } - // TimestampColumnArithmeticTimestampColumn.txt // TimestampScalarArithmeticTimestampColumn.txt // TimestampColumnArithmeticTimestampScalar.txt @@ -2955,10 +2832,23 @@ private void generateTimestampArithmeticTimestampBase(String[] tdesc) throws Exc private void generateTimestampArithmeticTimestamp(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; + String camelOperandType1 = getCamelCaseType(operandType1); String colOrScalar1 = tdesc[3]; String operandType2 = tdesc[4]; + String camelOperandType2 = getCamelCaseType(operandType2); String colOrScalar2 = tdesc[5]; + String returnType; + if (operandType1.equals(operandType2)) { + // timestamp - timestamp + // interval_day_time +/- interval_day_time + returnType = "interval_day_time"; + } else { + // timestamp +/- interval_day_time + // interval_day_time + timestamp + returnType = "timestamp"; + } + String className = getCamelCaseType(operandType1) + colOrScalar1 + operatorName + getCamelCaseType(operandType2) + colOrScalar2; String baseClassName = "Timestamp" + colOrScalar1 + operatorName + @@ -2971,20 +2861,26 @@ private void generateTimestampArithmeticTimestamp(String[] tdesc) throws Excepti String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); templateString = templateString.replaceAll("", baseClassName); + templateString = templateString.replaceAll("", operatorName.toLowerCase()); templateString = templateString.replaceAll("", operandType1); templateString = templateString.replaceAll("", operandType2); - if (colOrScalar1.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 1, operandType1); - } - if (colOrScalar2.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 2, operandType2); - } - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); + templateString = templateString.replaceAll("", camelOperandType1); + templateString = templateString.replaceAll("", camelOperandType2); + templateString = templateString.replaceAll("", getTimestampHiveType(operandType1)); + templateString = templateString.replaceAll("", getTimestampHiveType(operandType2)); String inputColumnVectorType1 = this.getColumnVectorType(operandType1); + templateString = templateString.replaceAll("", inputColumnVectorType1); String inputColumnVectorType2 = this.getColumnVectorType(operandType2); + templateString = templateString.replaceAll("", inputColumnVectorType2); + + String outputColumnVectorType = this.getColumnVectorType(returnType); + templateString = templateString.replaceAll("", outputColumnVectorType); + templateString = templateString.replaceAll("", getCamelCaseType(returnType)); + templateString = templateString.replaceAll("", returnType); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); /* UNDONE: Col Col, vs Scalar Col vs Col Scalar testCodeGen.addColumnColumnOperationTestCases( @@ -2995,30 +2891,6 @@ private void generateTimestampArithmeticTimestamp(String[] tdesc) throws Excepti */ } - // DateColumnArithmeticTimestampColumnBase.txt - // DateScalarArithmeticTimestampColumnBase.txt - // DateColumnArithmeticTimestampScalarBase.txt - // - private void generateDateArithmeticTimestampBase(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String colOrScalar1 = tdesc[2]; - String colOrScalar2 = tdesc[3]; - - String baseClassName = "Date" + colOrScalar1 + operatorName + - "Timestamp" + colOrScalar2 + "Base"; - - //Read the template into a string; - String fileName = "Date" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + - "Timestamp" + colOrScalar2 + "Base"; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", baseClassName); - templateString = templateString.replaceAll("", operatorName.toLowerCase()); - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - baseClassName, templateString); - } - // DateColumnArithmeticTimestampColumn.txt // DateScalarArithmeticTimestampColumn.txt // DateColumnArithmeticTimestampScalar.txt @@ -3026,14 +2898,23 @@ private void generateDateArithmeticTimestampBase(String[] tdesc) throws Exceptio private void generateDateArithmeticTimestamp(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; + String camelOperandType1 = getCamelCaseType(operandType1); String colOrScalar1 = tdesc[3]; String operandType2 = tdesc[4]; + String camelOperandType2 = getCamelCaseType(operandType2); String colOrScalar2 = tdesc[5]; - String className = getCamelCaseType(operandType1) + colOrScalar1 + operatorName + - getCamelCaseType(operandType2) + colOrScalar2; - String baseClassName = "Date" + colOrScalar1 + operatorName + - "Timestamp" + colOrScalar2 + "Base"; + String returnType; + if (operandType1.equals("interval_day_time") || operandType2.equals("interval_day_time")) { + returnType = "timestamp"; + } else if (operandType1.equals("timestamp") || operandType2.equals("timestamp")) { + returnType = "interval_day_time"; + } else { + returnType = "unknown"; + } + + String className = camelOperandType1 + colOrScalar1 + operatorName + + camelOperandType2 + colOrScalar2; //Read the template into a string; String fileName = "Date" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + @@ -3041,21 +2922,26 @@ private void generateDateArithmeticTimestamp(String[] tdesc) throws Exception { File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", baseClassName); + templateString = templateString.replaceAll("", operatorName.toLowerCase()); templateString = templateString.replaceAll("", operandType1); templateString = templateString.replaceAll("", operandType2); - if (colOrScalar1.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 1, operandType1); - } - if (colOrScalar2.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 2, operandType2); - } - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); + templateString = templateString.replaceAll("", camelOperandType1); + templateString = templateString.replaceAll("", camelOperandType2); + templateString = templateString.replaceAll("", getTimestampHiveType(operandType1)); + templateString = templateString.replaceAll("", getTimestampHiveType(operandType2)); String inputColumnVectorType1 = this.getColumnVectorType(operandType1); + templateString = templateString.replaceAll("", inputColumnVectorType1); String inputColumnVectorType2 = this.getColumnVectorType(operandType2); + templateString = templateString.replaceAll("", inputColumnVectorType2); + + String outputColumnVectorType = this.getColumnVectorType(returnType); + templateString = templateString.replaceAll("", outputColumnVectorType); + templateString = templateString.replaceAll("", getCamelCaseType(returnType)); + templateString = templateString.replaceAll("", returnType); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); /* UNDONE: Col Col, vs Scalar Col vs Col Scalar testCodeGen.addColumnColumnOperationTestCases( @@ -3066,30 +2952,6 @@ private void generateDateArithmeticTimestamp(String[] tdesc) throws Exception { */ } - // TimestampColumnArithmeticDateColumnBase.txt - // TimestampScalarArithmeticDateColumnBase.txt - // TimestampColumnArithmeticDateScalarBase.txt - // - private void generateTimestampArithmeticDateBase(String[] tdesc) throws Exception { - String operatorName = tdesc[1]; - String colOrScalar1 = tdesc[2]; - String colOrScalar2 = tdesc[3]; - - String baseClassName = "Timestamp" + colOrScalar1 + operatorName + - "Date" + colOrScalar2 + "Base"; - - //Read the template into a string; - String fileName = "Timestamp" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + - "Date" + colOrScalar2 + "Base"; - File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); - String templateString = readFile(templateFile); - templateString = templateString.replaceAll("", baseClassName); - templateString = templateString.replaceAll("", operatorName.toLowerCase()); - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - baseClassName, templateString); - } - // TimestampColumnArithmeticDateColumn.txt // TimestampScalarArithmeticDateColumn.txt // TimestampColumnArithmeticDateScalar.txt @@ -3097,14 +2959,23 @@ private void generateTimestampArithmeticDateBase(String[] tdesc) throws Exceptio private void generateTimestampArithmeticDate(String[] tdesc) throws Exception { String operatorName = tdesc[1]; String operandType1 = tdesc[2]; + String camelOperandType1 = getCamelCaseType(operandType1); String colOrScalar1 = tdesc[3]; String operandType2 = tdesc[4]; + String camelOperandType2 = getCamelCaseType(operandType2); String colOrScalar2 = tdesc[5]; - String className = getCamelCaseType(operandType1) + colOrScalar1 + operatorName + - getCamelCaseType(operandType2) + colOrScalar2; - String baseClassName = "Timestamp" + colOrScalar1 + operatorName + - "Date" + colOrScalar2 + "Base"; + String returnType; + if (operandType1.equals("interval_day_time") || operandType2.equals("interval_day_time")) { + returnType = "timestamp"; + } else if (operandType1.equals("timestamp") || operandType2.equals("timestamp")) { + returnType = "interval_day_time"; + } else { + returnType = "unknown"; + } + + String className = camelOperandType1 + colOrScalar1 + operatorName + + camelOperandType2 + colOrScalar2; //Read the template into a string; String fileName = "Timestamp" + (colOrScalar1.equals("Col") ? "Column" : colOrScalar1) + "Arithmetic" + @@ -3112,21 +2983,26 @@ private void generateTimestampArithmeticDate(String[] tdesc) throws Exception { File templateFile = new File(joinPath(this.expressionTemplateDirectory, fileName + ".txt")); String templateString = readFile(templateFile); templateString = templateString.replaceAll("", className); - templateString = templateString.replaceAll("", baseClassName); + templateString = templateString.replaceAll("", operatorName.toLowerCase()); templateString = templateString.replaceAll("", operandType1); templateString = templateString.replaceAll("", operandType2); - if (colOrScalar1.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 1, operandType1); - } - if (colOrScalar2.equals("Scalar")) { - templateString = replaceTimestampScalar(templateString, 2, operandType2); - } - - writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, - className, templateString); + templateString = templateString.replaceAll("", camelOperandType1); + templateString = templateString.replaceAll("", camelOperandType2); + templateString = templateString.replaceAll("", getTimestampHiveType(operandType1)); + templateString = templateString.replaceAll("", getTimestampHiveType(operandType2)); String inputColumnVectorType1 = this.getColumnVectorType(operandType1); + templateString = templateString.replaceAll("", inputColumnVectorType1); String inputColumnVectorType2 = this.getColumnVectorType(operandType2); + templateString = templateString.replaceAll("", inputColumnVectorType2); + + String outputColumnVectorType = this.getColumnVectorType(returnType); + templateString = templateString.replaceAll("", outputColumnVectorType); + templateString = templateString.replaceAll("", getCamelCaseType(returnType)); + templateString = templateString.replaceAll("", returnType); + + writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory, + className, templateString); /* UNDONE: Col Col, vs Scalar Col vs Col Scalar testCodeGen.addColumnColumnOperationTestCases( @@ -3272,8 +3148,10 @@ private String getColumnVectorType(String primitiveType) throws Exception { return "DecimalColumnVector"; } else if (primitiveType.equals("string")) { return "BytesColumnVector"; - } else if (isTimestampIntervalType(primitiveType)) { + } else if (primitiveType.equals("timestamp")) { return "TimestampColumnVector"; + } else if (primitiveType.equals("interval_day_time")) { + return "IntervalDayTimeColumnVector"; } throw new Exception("Unimplemented primitive column vector type: " + primitiveType); } diff --git common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java deleted file mode 100644 index e262f01..0000000 --- common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalDayTime.java +++ /dev/null @@ -1,245 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.math.BigDecimal; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.lang.builder.HashCodeBuilder; -import org.apache.hive.common.util.DateUtils; - -/** - * Day-time interval type representing an offset in days/hours/minutes/seconds, - * with nanosecond precision. - * 1 day = 24 hours = 1440 minutes = 86400 seconds - */ -public class HiveIntervalDayTime implements Comparable { - - // days/hours/minutes/seconds all represented as seconds - protected long totalSeconds; - protected int nanos; - - public HiveIntervalDayTime() { - } - - public HiveIntervalDayTime(int days, int hours, int minutes, int seconds, int nanos) { - set(days, hours, minutes, seconds, nanos); - } - - public HiveIntervalDayTime(long seconds, int nanos) { - set(seconds, nanos); - } - - public HiveIntervalDayTime(BigDecimal seconds) { - set(seconds); - } - - public HiveIntervalDayTime(HiveIntervalDayTime other) { - set(other.totalSeconds, other.nanos); - } - - public int getDays() { - return (int) TimeUnit.SECONDS.toDays(totalSeconds); - } - - public int getHours() { - return (int) (TimeUnit.SECONDS.toHours(totalSeconds) % TimeUnit.DAYS.toHours(1)); - } - - public int getMinutes() { - return (int) (TimeUnit.SECONDS.toMinutes(totalSeconds) % TimeUnit.HOURS.toMinutes(1)); - } - - public int getSeconds() { - return (int) (totalSeconds % TimeUnit.MINUTES.toSeconds(1)); - } - - public int getNanos() { - return nanos; - } - - /** - * Returns days/hours/minutes all converted into seconds. - * Nanos still need to be retrieved using getNanos() - * @return - */ - public long getTotalSeconds() { - return totalSeconds; - } - - /** - * Ensures that the seconds and nanoseconds fields have consistent sign - */ - protected void normalizeSecondsAndNanos() { - if (totalSeconds > 0 && nanos < 0) { - --totalSeconds; - nanos += DateUtils.NANOS_PER_SEC; - } else if (totalSeconds < 0 && nanos > 0) { - ++totalSeconds; - nanos -= DateUtils.NANOS_PER_SEC; - } - } - - public void set(int days, int hours, int minutes, int seconds, int nanos) { - long totalSeconds = seconds; - totalSeconds += TimeUnit.DAYS.toSeconds(days); - totalSeconds += TimeUnit.HOURS.toSeconds(hours); - totalSeconds += TimeUnit.MINUTES.toSeconds(minutes); - totalSeconds += TimeUnit.NANOSECONDS.toSeconds(nanos); - nanos = nanos % DateUtils.NANOS_PER_SEC; - - this.totalSeconds = totalSeconds; - this.nanos = nanos; - - normalizeSecondsAndNanos(); - } - - public void set(long seconds, int nanos) { - this.totalSeconds = seconds; - this.nanos = nanos; - normalizeSecondsAndNanos(); - } - - public void set(PisaTimestamp pisaTimestamp) { - this.totalSeconds = pisaTimestamp.getEpochSeconds(); - this.nanos = pisaTimestamp.getSignedNanos(); - normalizeSecondsAndNanos(); - } - - public void set(BigDecimal totalSecondsBd) { - long totalSeconds = totalSecondsBd.longValue(); - BigDecimal fractionalSecs = totalSecondsBd.remainder(BigDecimal.ONE); - int nanos = fractionalSecs.multiply(DateUtils.NANOS_PER_SEC_BD).intValue(); - set(totalSeconds, nanos); - } - - public void set(HiveIntervalDayTime other) { - set(other.getTotalSeconds(), other.getNanos()); - } - - public HiveIntervalDayTime negate() { - return new HiveIntervalDayTime(-getTotalSeconds(), -getNanos()); - } - - public PisaTimestamp pisaTimestampUpdate(PisaTimestamp pisaTimestamp) { - // NOTE: Our nanos here are *SIGNED*. - return pisaTimestamp.updateFromEpochSecondsAndSignedNanos(totalSeconds, nanos); - } - - @Override - public int compareTo(HiveIntervalDayTime other) { - long cmp = this.totalSeconds - other.totalSeconds; - if (cmp == 0) { - cmp = this.nanos - other.nanos; - } - if (cmp != 0) { - cmp = cmp > 0 ? 1 : -1; - } - return (int) cmp; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof HiveIntervalDayTime)) { - return false; - } - return 0 == compareTo((HiveIntervalDayTime) obj); - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(totalSeconds).append(nanos).toHashCode(); - } - - @Override - public String toString() { - // If normalize() was used, then day-hour-minute-second-nanos should have the same sign. - // This is currently working with that assumption. - boolean isNegative = (totalSeconds < 0 || nanos < 0); - String daySecondSignStr = isNegative ? "-" : ""; - - return String.format("%s%d %02d:%02d:%02d.%09d", - daySecondSignStr, Math.abs(getDays()), - Math.abs(getHours()), Math.abs(getMinutes()), - Math.abs(getSeconds()), Math.abs(getNanos())); - } - - public static HiveIntervalDayTime valueOf(String strVal) { - HiveIntervalDayTime result = null; - if (strVal == null) { - throw new IllegalArgumentException("Interval day-time string was null"); - } - Matcher patternMatcher = PATTERN_MATCHER.get(); - patternMatcher.reset(strVal); - if (patternMatcher.matches()) { - // Parse out the individual parts - try { - // Sign - whether interval is positive or negative - int sign = 1; - String field = patternMatcher.group(1); - if (field != null && field.equals("-")) { - sign = -1; - } - int days = sign * - DateUtils.parseNumericValueWithRange("day", patternMatcher.group(2), - 0, Integer.MAX_VALUE); - byte hours = (byte) (sign * - DateUtils.parseNumericValueWithRange("hour", patternMatcher.group(3), 0, 23)); - byte minutes = (byte) (sign * - DateUtils.parseNumericValueWithRange("minute", patternMatcher.group(4), 0, 59)); - int seconds = 0; - int nanos = 0; - field = patternMatcher.group(5); - if (field != null) { - BigDecimal bdSeconds = new BigDecimal(field); - if (bdSeconds.compareTo(DateUtils.MAX_INT_BD) > 0) { - throw new IllegalArgumentException("seconds value of " + bdSeconds + " too large"); - } - seconds = sign * bdSeconds.intValue(); - nanos = sign * bdSeconds.subtract(new BigDecimal(bdSeconds.toBigInteger())) - .multiply(DateUtils.NANOS_PER_SEC_BD).intValue(); - } - - result = new HiveIntervalDayTime(days, hours, minutes, seconds, nanos); - } catch (Exception err) { - throw new IllegalArgumentException("Error parsing interval day-time string: " + strVal, err); - } - } else { - throw new IllegalArgumentException( - "Interval string does not match day-time format of 'd h:m:s.n': " + strVal); - } - - return result; - } - - // Simple pattern: D H:M:S.nnnnnnnnn - private final static String PARSE_PATTERN = - "([+|-])?(\\d+) (\\d+):(\\d+):((\\d+)(\\.(\\d+))?)"; - - private static final ThreadLocal PATTERN_MATCHER = new ThreadLocal() { - @Override - protected Matcher initialValue() { - return Pattern.compile(PARSE_PATTERN).matcher(""); - } - }; -} diff --git common/src/java/org/apache/hive/common/util/DateUtils.java common/src/java/org/apache/hive/common/util/DateUtils.java index c749bcb..959a542 100644 --- common/src/java/org/apache/hive/common/util/DateUtils.java +++ common/src/java/org/apache/hive/common/util/DateUtils.java @@ -21,8 +21,6 @@ import java.math.BigDecimal; import java.text.SimpleDateFormat; -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; - /** * DateUtils. Thread-safe class * @@ -56,21 +54,4 @@ public static int parseNumericValueWithRange(String fieldName, } return result; } - - public static long getIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime) { - return intervalDayTime.getTotalSeconds() * NANOS_PER_SEC + intervalDayTime.getNanos(); - } - - public static void setIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime, - long totalNanos) { - intervalDayTime.set(totalNanos / NANOS_PER_SEC, (int) (totalNanos % NANOS_PER_SEC)); - } - - public static long getIntervalDayTimeTotalSecondsFromTotalNanos(long totalNanos) { - return totalNanos / NANOS_PER_SEC; - } - - public static int getIntervalDayTimeNanosFromTotalNanos(long totalNanos) { - return (int) (totalNanos % NANOS_PER_SEC); - } } diff --git data/files/timestamps.txt data/files/timestamps.txt new file mode 100644 index 0000000..36ffd23 --- /dev/null +++ data/files/timestamps.txt @@ -0,0 +1,50 @@ +6631-11-13 16:31:29.702202248 +6731-02-12 08:12:48.287783702 +6705-09-28 18:27:28.000845672 +5397-07-13 07:12:32.000896438 +9209-11-11 04:08:58.223768453 +9403-01-09 18:12:33.547 +6482-04-27 12:07:38.073915413 +7503-06-23 23:14:17.486 +1883-04-17 04:14:34.647766229 +0004-09-22 18:26:29.519542222 +7160-12-02 06:00:24.81200852 +8422-07-22 03:21:45.745036084 +4143-07-08 10:53:27.252802259 +5344-10-04 18:40:08.165 +5966-07-09 03:30:50.597 +9075-06-13 16:20:09.218517797 +1815-05-06 00:12:37.543584705 +7409-09-07 23:33:32.459349602 +5339-02-01 14:10:01.085678691 +4966-12-04 09:30:55.202 +1319-02-02 16:31:57.778 +1404-07-23 15:32:16.059185026 +6229-06-28 02:54:28.970117179 +0528-10-27 08:15:18.941718273 +8521-01-16 20:42:05.668832388 +1976-05-06 00:42:30.910786948 +2003-09-23 22:33:17.00003252 +2007-02-09 05:17:29.368756876 +1998-10-16 20:05:29.397591987 +1976-03-03 04:54:33.000895162 +1985-07-20 09:30:11.0 +2021-09-24 03:18:32.413655165 +2013-04-07 02:44:43.00086821 +2002-05-10 05:29:48.990818073 +1973-04-17 06:30:38.596784156 +1987-02-21 19:48:29.0 +1981-11-15 23:03:10.999338387 +2000-12-18 08:42:30.000595596 +1999-10-03 16:59:10.396903939 +2024-11-11 16:42:41.101 +2013-04-10 00:43:46.854731546 +2010-04-08 02:43:35.861742727 +2004-03-07 20:14:13.0 +1987-05-28 13:52:07.900916635 +1978-08-05 14:41:05.501 +1966-08-16 13:36:50.183618031 +2009-01-21 10:49:07.108 +1981-04-25 09:01:12.077192689 +1985-11-18 16:37:54.0 +1974-10-04 17:21:03.989 diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index f991d49..9fe5cad 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -285,6 +285,7 @@ minitez.query.files.shared=acid_globallimit.q,\ vector_inner_join.q,\ vector_interval_1.q,\ vector_interval_2.q,\ + vector_interval_arithmetic.q,\ vector_interval_mapjoin.q,\ vector_join30.q,\ vector_join_filters.q,\ @@ -315,6 +316,7 @@ minitez.query.files.shared=acid_globallimit.q,\ vector_reduce3.q,\ vector_string_concat.q,\ vector_struct_in.q,\ + vectorized_timestamp.q,\ vector_varchar_4.q,\ vector_varchar_mapjoin1.q,\ vector_varchar_simple.q,\ diff --git orc/src/java/org/apache/orc/impl/WriterImpl.java orc/src/java/org/apache/orc/impl/WriterImpl.java index ecac520..11a3fca 100644 --- orc/src/java/org/apache/orc/impl/WriterImpl.java +++ orc/src/java/org/apache/orc/impl/WriterImpl.java @@ -1734,19 +1734,17 @@ void writeBatch(ColumnVector vector, int offset, int length) throws IOException { super.writeBatch(vector, offset, length); TimestampColumnVector vec = (TimestampColumnVector) vector; + Timestamp val; if (vector.isRepeating) { if (vector.noNulls || !vector.isNull[0]) { - long millis = vec.getEpochMilliseconds(0); - int adjustedNanos = vec.getSignedNanos(0); - if (adjustedNanos < 0) { - adjustedNanos += NANOS_PER_SECOND; - } + val = vec.asScratchTimestamp(0); + long millis = val.getTime(); indexStatistics.updateTimestamp(millis); if (createBloomFilter) { bloomFilter.addLong(millis); } - final long secs = vec.getEpochSeconds(0) - base_timestamp; - final long nano = formatNanos(adjustedNanos); + final long secs = millis / MILLIS_PER_SECOND - base_timestamp; + final long nano = formatNanos(val.getNanos()); for(int i=0; i < length; ++i) { seconds.write(secs); nanos.write(nano); @@ -1755,14 +1753,11 @@ void writeBatch(ColumnVector vector, int offset, } else { for(int i=0; i < length; ++i) { if (vec.noNulls || !vec.isNull[i + offset]) { - long secs = vec.getEpochSeconds(i + offset) - base_timestamp; - long millis = vec.getEpochMilliseconds(i + offset); - int adjustedNanos = vec.getSignedNanos(i + offset); - if (adjustedNanos < 0) { - adjustedNanos += NANOS_PER_SECOND; - } + val = vec.asScratchTimestamp(i + offset); + long millis = val.getTime(); + long secs = millis / MILLIS_PER_SECOND - base_timestamp; seconds.write(secs); - nanos.write(formatNanos(adjustedNanos)); + nanos.write(formatNanos(val.getNanos())); indexStatistics.updateTimestamp(millis); if (createBloomFilter) { bloomFilter.addLong(millis); diff --git ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt index 845bc5f..c3d8d7e 100644 --- ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthColumn.txt @@ -18,15 +18,18 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** - * Generated from template DateColumnArithmeticIntervalYearMonthColumn.txt, which covers binary arithmetic + * Generated from template DateColumnArithmeticIntervalYearMonthColumn.txt, which covers binary arithmetic * expressions between date and interval year month columns. */ public class extends VectorExpression { @@ -36,12 +39,18 @@ public class extends VectorExpression { private int colNum1; private int colNum2; private int outputColumn; + private Date scratchDate1; + private HiveIntervalYearMonth scratchIntervalYearMonth2; + private Date outputDate; private DateTimeMath dtm = new DateTimeMath(); public (int colNum1, int colNum2, int outputColumn) { this.colNum1 = colNum1; this.colNum2 = colNum2; this.outputColumn = outputColumn; + scratchDate1 = new Date(0); + scratchIntervalYearMonth2 = new HiveIntervalYearMonth(); + outputDate = new Date(0); } public () { @@ -54,10 +63,10 @@ public class extends VectorExpression { super.evaluateChildren(batch); } - // Input #1 is type date (epochDays). + // Input #1 is type date. LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - // Input #2 is type interval_year_month (months). + // Input #2 is type interval_year_month. LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; // Output is type date. @@ -89,38 +98,65 @@ public class extends VectorExpression { * conditional checks in the inner loop. */ if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = dtm.addMonthsToDays(vector1[0], (int) vector2[0]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[0])); + scratchIntervalYearMonth2.set((int) vector2[0]); + dtm.( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[0] = DateWritable.dateToDays(outputDate); } else if (inputColVector1.isRepeating) { + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[0])); if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector1[0], (int) vector2[i]); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector1[0], (int) vector2[i]); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else if (inputColVector2.isRepeating) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector1[i], (int) vector2[0]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector1[i], (int) vector2[0]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector1[i], (int) vector2[i]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector1[i], (int) vector2[i]); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + scratchDate1, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } diff --git ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt index 86a95c9..d1474fb 100644 --- ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticIntervalYearMonthScalar.txt @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -25,6 +27,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** * Generated from template DateColumnArithmeticIntervalYearMonthScalar.txt, which covers binary arithmetic @@ -35,14 +38,18 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; private int colNum; - private long value; + private HiveIntervalYearMonth value; private int outputColumn; + private Date scratchDate1; + private Date outputDate; private DateTimeMath dtm = new DateTimeMath(); public (int colNum, long value, int outputColumn) { this.colNum = colNum; - this.value = value; + this.value = new HiveIntervalYearMonth((int) value); this.outputColumn = outputColumn; + scratchDate1 = new Date(0); + outputDate = new Date(0); } public () { @@ -55,19 +62,19 @@ public class extends VectorExpression { super.evaluateChildren(batch); } - // Input #1 is type date (epochDays). - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; + // Input #1 is type date. + LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum]; // Output is type date. LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; + boolean[] inputIsNull = inputColVector1.isNull; boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - outputColVector.isRepeating = inputColVector.isRepeating; + outputColVector.noNulls = inputColVector1.noNulls; + outputColVector.isRepeating = inputColVector1.isRepeating; int n = batch.size; - long[] vector = inputColVector.vector; + long[] vector1 = inputColVector1.vector; long[] outputVector = outputColVector.vector; // return immediately if batch is empty @@ -75,32 +82,46 @@ public class extends VectorExpression { return; } - if (inputColVector.isRepeating) { - outputVector[0] = dtm.addMonthsToDays(vector[0], (int) value); - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. + if (inputColVector1.isRepeating) { + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[0])); + dtm.( + scratchDate1, value, outputDate); + outputVector[0] = DateWritable.dateToDays(outputDate); + // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector.noNulls) { + } else if (inputColVector1.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector[i], (int) value); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchDate1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector[i], (int) value); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchDate1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else /* there are nulls */ { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector[i], (int) value); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchDate1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector[i], (int) value); + scratchDate1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchDate1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumn.txt index 6241ee2..63cebaf 100644 --- ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumn.txt @@ -18,28 +18,155 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** - * Generated from template DateColumnArithmeticTimestampColumn.txt, which covers binary arithmetic - * expressions between a date column and a timestamp column. + * Generated from template DateColumnArithmeticTimestampColumn.txt, a class + * which covers binary arithmetic expressions between a date column and timestamp column. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; + private int colNum1; + private int colNum2; + private int outputColumn; + private Timestamp scratchTimestamp1; + private DateTimeMath dtm = new DateTimeMath(); + public (int colNum1, int colNum2, int outputColumn) { - super(colNum1, colNum2, outputColumn); + this.colNum1 = colNum1; + this.colNum2 = colNum2; + this.outputColumn = outputColumn; + scratchTimestamp1 = new Timestamp(0); } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type Date (days). For the math we convert it to a timestamp. + LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; + + // Input #2 is type . + inputColVector2 = () batch.cols[colNum2]; + + // Output is type . + outputColVector = () batch.cols[outputColumn]; + + int[] sel = batch.selected; + int n = batch.size; + long[] vector1 = inputColVector1.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + outputColVector.isRepeating = + inputColVector1.isRepeating && inputColVector2.isRepeating + || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] + || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; + + // Handle nulls first + NullUtil.propagateNullsColCol( + inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); + + /* Disregard nulls for processing. In other words, + * the arithmetic operation is performed even if one or + * more inputs are null. This is to improve speed by avoiding + * conditional checks in the inner loop. + */ + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0])); + dtm.( + scratchTimestamp1, inputColVector2.asScratch(0), outputColVector.getScratch()); + outputColVector.setFromScratch(0); + } else if (inputColVector1.isRepeating) { + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0])); + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + scratchTimestamp1, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + scratchTimestamp1, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else if (inputColVector2.isRepeating) { + value2 = inputColVector2.asScratch(0); + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchTimestamp1, value2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchTimestamp1, value2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchTimestamp1, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchTimestamp1, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } + + /* For the case when the output can have null values, follow + * the convention that the data values must be 1 for long and + * NaN for double. This is to prevent possible later zero-divide errors + * in complex arithmetic expressions like col2 / (col1 - 1) + * in the case when some col1 entries are null. + */ + NullUtil.setNullDataEntries(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; } @Override @@ -49,7 +176,7 @@ public class extends { VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType(""), + VectorExpressionDescriptor.ArgumentType.getType("date"), VectorExpressionDescriptor.ArgumentType.getType("")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, diff --git ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumnBase.txt deleted file mode 100644 index a61b769..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampColumnBase.txt +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.serde2.io.DateWritable; - -/** - * Generated from template DateColumnArithmeticTimestampColumnBase.txt, a base class - * which covers binary arithmetic expressions between a date column and timestamp column. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum1; - private int colNum2; - private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; - - public (int colNum1, int colNum2, int outputColumn) { - this.colNum1 = colNum1; - this.colNum2 = colNum2; - this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - // Input #1 is type Date (epochDays). - LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - - // Input #2 is type timestamp/interval_day_time. - TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2]; - - // Output is type timestamp. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; - - int[] sel = batch.selected; - int n = batch.size; - long[] vector1 = inputColVector1.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - outputColVector.isRepeating = - inputColVector1.isRepeating && inputColVector2.isRepeating - || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] - || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; - - // Handle nulls first - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])), - inputColVector2.asScratchPisaTimestamp(0), - 0); - } else if (inputColVector1.isRepeating) { - PisaTimestamp value1 = - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])); - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - value1, - inputColVector2.asScratchPisaTimestamp(i), - i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - value1, - inputColVector2.asScratchPisaTimestamp(i), - i); - } - } - } else if (inputColVector2.isRepeating) { - PisaTimestamp value2 = inputColVector2.asScratchPisaTimestamp(0); - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - value2, - i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - value2, - i); - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - inputColVector2.asScratchPisaTimestamp(i), - i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - inputColVector2.asScratchPisaTimestamp(i), - i); - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesTimestamp(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "timestamp"; - } -} - diff --git ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalar.txt ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalar.txt index b813d11..7aee529 100644 --- ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalar.txt @@ -19,32 +19,123 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hive.common.util.DateUtils; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** - * Generated from template DateColumnArithmeticTimestampScalar.txt, which covers binary arithmetic - * expressions between a date column and a timestamp scalar. + * Generated from template DateColumnArithmeticTimestampScalarBase.txt, a base class + * which covers binary arithmetic expressions between a date column and a timestamp scalar. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; - public (int colNum, value, int outputColumn) { - super(colNum, , outputColumn); + private int colNum; + private value; + private int outputColumn; + private Timestamp scratchTimestamp1; + private DateTimeMath dtm = new DateTimeMath(); + + public (int colNum, value, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; + scratchTimestamp1 = new Timestamp(0); } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type date (days). For the math we convert it to a timestamp. + LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum]; + + // Output is type . + outputColVector = () batch.cols[outputColumn]; + + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector1.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector1.noNulls; + outputColVector.isRepeating = inputColVector1.isRepeating; + int n = batch.size; + long[] vector1 = inputColVector1.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector1.isRepeating) { + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0])); + dtm.( + scratchTimestamp1, value, outputColVector.getScratch()); + outputColVector.setFromScratch(0); + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + } else if (inputColVector1.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchTimestamp1, value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchTimestamp1, value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchTimestamp1, value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.( + scratchTimestamp1, value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + } + + NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; } @Override @@ -54,7 +145,7 @@ public class extends { VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType(""), + VectorExpressionDescriptor.ArgumentType.getType("date"), VectorExpressionDescriptor.ArgumentType.getType("")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, diff --git ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalarBase.txt ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalarBase.txt deleted file mode 100644 index d64fba0..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/DateColumnArithmeticTimestampScalarBase.txt +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.serde2.io.DateWritable; - -/** - * Generated from template DateColumnArithmeticTimestampScalarBase.txt, a base class - * which covers binary arithmetic expressions between a date column and a timestamp scalar. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; - - public (int colNum, PisaTimestamp value, int outputColumn) { - this.colNum = colNum; - this.value = value; - this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - // Input #1 is type date (epochDays). - LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum]; - - // Output is type timestamp. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; - - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector1.isNull; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector1.noNulls; - outputColVector.isRepeating = inputColVector1.isRepeating; - int n = batch.size; - long[] vector1 = inputColVector1.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector1.isRepeating) { - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])), - value, - 0); - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector1.noNulls) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - value, - i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - value, - i); - } - } - } else /* there are nulls */ { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - value, - i); - outputIsNull[i] = inputIsNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - value, - i); - } - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - } - } - - NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "timestamp"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticIntervalYearMonthColumn.txt ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticIntervalYearMonthColumn.txt index 653565e..c68ac34 100644 --- ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticIntervalYearMonthColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticIntervalYearMonthColumn.txt @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -33,6 +35,7 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** * Generated from template DateTimeScalarArithmeticIntervalYearMonthColumn.txt. @@ -44,14 +47,18 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; private int colNum; - private long value; + private Date value; private int outputColumn; + private HiveIntervalYearMonth scratchIntervalYearMonth2; + private Date outputDate; private DateTimeMath dtm = new DateTimeMath(); public (long value, int colNum, int outputColumn) { this.colNum = colNum; - this.value = value; + this.value = new Date(DateWritable.daysToMillis((int) value)); this.outputColumn = outputColumn; + scratchIntervalYearMonth2 = new HiveIntervalYearMonth(); + outputDate = new Date(0); } public () { @@ -70,18 +77,18 @@ public class extends VectorExpression { } // Input #2 is type Interval_Year_Month (months). - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; + LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum]; // Output is type Date. LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; + boolean[] inputIsNull = inputColVector2.isNull; boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - outputColVector.isRepeating = inputColVector.isRepeating; + outputColVector.noNulls = inputColVector2.noNulls; + outputColVector.isRepeating = inputColVector2.isRepeating; int n = batch.size; - long[] vector = inputColVector.vector; + long[] vector2 = inputColVector2.vector; long[] outputVector = outputColVector.vector; // return immediately if batch is empty @@ -89,32 +96,46 @@ public class extends VectorExpression { return; } - if (inputColVector.isRepeating) { - outputVector[0] = dtm.addMonthsToDays(value, (int) vector[0]); - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. + if (inputColVector2.isRepeating) { + scratchIntervalYearMonth2.set((int) vector2[0]); + dtm.( + value, scratchIntervalYearMonth2, outputDate); + outputVector[0] = DateWritable.dateToDays(outputDate); + // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector.noNulls) { + } else if (inputColVector2.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(value, (int) vector[i]); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(value, (int) vector[i]); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else { /* there are nulls */ if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(value, (int) vector[i]); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(value, (int) vector[i]); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value, scratchIntervalYearMonth2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumn.txt index e93bed5..cb6b750 100644 --- ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumn.txt @@ -18,45 +18,141 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.common.type.PisaTimestamp; /* * Because of the templatized nature of the code, either or both * of these ColumnVector imports may be needed. Listing both of them * rather than using ....vectorization.*; */ -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** - * Generated from template DateScalarArithmeticTimestampColumn.txt. + * Generated from template DateTimeScalarArithmeticTimestampColumnBase.txt. * Implements a vectorized arithmetic operator with a scalar on the left and a * column vector on the right. The result is output to an output column vector. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; + private int colNum; + private Timestamp value; + private int outputColumn; + private DateTimeMath dtm = new DateTimeMath(); + public (long value, int colNum, int outputColumn) { - super(value, colNum, outputColumn); + this.colNum = colNum; + // Scalar input #1 is type date (days). For the math we convert it to a timestamp. + this.value = new Timestamp(0); + this.value.setTime(DateWritable.daysToMillis((int) value)); + this.outputColumn = outputColumn; } public () { } @Override + /** + * Method to evaluate scalar-column operation in vectorized fashion. + * + * @batch a package of rows with each column stored in a vector + */ + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #2 is type . + inputColVector2 = () batch.cols[colNum]; + + // Output is type . + outputColVector = () batch.cols[outputColumn]; + + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector2.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector2.noNulls; + outputColVector.isRepeating = inputColVector2.isRepeating; + int n = batch.size; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector2.isRepeating) { + dtm.( + value, inputColVector2.asScratch(0), outputColVector.getScratch()); + outputColVector.setFromScratch(0); + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + } else if (inputColVector2.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + value, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + value, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else { /* there are nulls */ + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + value, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + value, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + } + + NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; + } + + @Override public VectorExpressionDescriptor.Descriptor getDescriptor() { return (new VectorExpressionDescriptor.Builder()) .setMode( VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType(""), + VectorExpressionDescriptor.ArgumentType.getType("date"), VectorExpressionDescriptor.ArgumentType.getType("")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.SCALAR, diff --git ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumnBase.txt deleted file mode 100644 index a1f4e6f..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/DateScalarArithmeticTimestampColumnBase.txt +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.*; - -import org.apache.hadoop.hive.common.type.PisaTimestamp; -/* - * Because of the templatized nature of the code, either or both - * of these ColumnVector imports may be needed. Listing both of them - * rather than using ....vectorization.*; - */ -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.serde2.io.DateWritable; - -/** - * Generated from template DateTimeScalarArithmeticTimestampColumnBase.txt. - * Implements a vectorized arithmetic operator with a scalar on the left and a - * column vector on the right. The result is output to an output column vector. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - private int outputColumn; - - public (long value, int colNum, int outputColumn) { - this.colNum = colNum; - this.value = new PisaTimestamp().updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) value)); - this.outputColumn = outputColumn; - } - - public () { - } - - @Override - /** - * Method to evaluate scalar-column operation in vectorized fashion. - * - * @batch a package of rows with each column stored in a vector - */ - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - // Input #2 is type timestamp/interval_day_time. - TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum]; - - // Output is type timestamp. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; - - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector2.isNull; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector2.noNulls; - outputColVector.isRepeating = inputColVector2.isRepeating; - int n = batch.size; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector2.isRepeating) { - outputColVector.( - value, - inputColVector2.asScratchPisaTimestamp(0), - 0); - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector2.noNulls) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - value, - inputColVector2.asScratchPisaTimestamp(i), - i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - value, - inputColVector2.asScratchPisaTimestamp(i), - i); - } - } - } else { /* there are nulls */ - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - value, - inputColVector2.asScratchPisaTimestamp(i), - i); - outputIsNull[i] = inputIsNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - value, - inputColVector2.asScratchPisaTimestamp(i), - i); - } - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - } - } - - NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "timestamp"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeColumn.txt deleted file mode 100644 index 8d9bdf1..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeColumn.txt +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - -/** - * Generated from template FilterIntervalDayTimeColumnCompareColumn.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type, however output is not - * produced in a separate column. - * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering. - */ -public class extends { - - public (int colNum1, int colNum2) { - super(colNum1, colNum2); - } - - public () { - super(); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.FILTER) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"), - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeScalar.txt ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeScalar.txt deleted file mode 100644 index 7022b4f..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeColumnCompareIntervalDayTimeScalar.txt +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - -/** - * Generated from template FilterIntervalDayTimeColumnCompareScalar.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type, however output is not - * produced in a separate column. - * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering. - */ -public class extends { - - public (int colNum, HiveIntervalDayTime value) { - super(colNum, value.pisaTimestampUpdate(new PisaTimestamp())); - } - - public () { - super(); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.FILTER) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"), - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeScalarCompareIntervalDayTimeColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeScalarCompareIntervalDayTimeColumn.txt deleted file mode 100644 index d227bf0..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/FilterIntervalDayTimeScalarCompareIntervalDayTimeColumn.txt +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - -/** - * Generated from template FilterIntervalDayTimeScalarCompareColumn.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type, however output is not - * produced in a separate column. - * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering. - */ -public class extends { - - public (HiveIntervalDayTime value, int colNum) { - super(value.pisaTimestampUpdate(new PisaTimestamp()), colNum); - } - - public () { - super(); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.FILTER) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"), - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.SCALAR, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampColumn.txt index 0c8321f..57caf7e 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampColumn.txt @@ -19,8 +19,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampScalar.txt ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampScalar.txt index 7e4d55e..1b86691 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleColumnCompareTimestampScalar.txt @@ -19,8 +19,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -36,7 +36,7 @@ public class extends { private static final long serialVersionUID = 1L; public (int colNum, Timestamp value) { - super(colNum, new PisaTimestamp(value).()); + super(colNum, TimestampColumnVector.(value)); } public () { diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleScalarCompareTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleScalarCompareTimestampColumn.txt index ba6ca66..f5f59c2 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleScalarCompareTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterLongDoubleScalarCompareTimestampColumn.txt @@ -18,6 +18,10 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; + import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnBetween.txt ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnBetween.txt index 12f73da..4298d79 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnBetween.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnBetween.txt @@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -39,14 +38,14 @@ public class extends VectorExpression { private int colNum; // The comparison is of the form "column BETWEEN leftValue AND rightValue" - private PisaTimestamp leftValue; - private PisaTimestamp rightValue; - private PisaTimestamp scratchValue; + private Timestamp leftValue; + private Timestamp rightValue; + private Timestamp scratchValue; public (int colNum, Timestamp leftValue, Timestamp rightValue) { this.colNum = colNum; - this.leftValue = new PisaTimestamp(leftValue); - this.rightValue = new PisaTimestamp(rightValue); + this.leftValue = leftValue; + this.rightValue = rightValue; } public () { diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumn.txt index 746b297..31dce1c 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumn.txt @@ -18,22 +18,421 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; /** - * Generated from template FilterTimestampColumnCompareTimestampColumn.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type, however output is not - * produced in a separate column. + * Generated from template FilterTimestampColumnCompareColumn.txt, which covers binary comparison + * filter expressions between two columns. Output is not produced in a separate column. * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering. */ -public class extends { +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; - public (int colNum1, int colNum2) { - super(colNum1, colNum2); + private int colNum1; + private int colNum2; + + public (int colNum1, int colNum2) { + this.colNum1 = colNum1; + this.colNum2 = colNum2; } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type . + inputColVector1 = () batch.cols[colNum1]; + + // Input #2 is type . + inputColVector2 = () batch.cols[colNum2]; + + int[] sel = batch.selected; + boolean[] nullPos1 = inputColVector1.isNull; + boolean[] nullPos2 = inputColVector2.isNull; + int n = batch.size; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + // handle case where neither input has nulls + if (inputColVector1.noNulls && inputColVector2.noNulls) { + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + + /* Either all must remain selected or all will be eliminated. + * Repeating property will not change. + */ + if (!(inputColVector1.compareTo(0, inputColVector2, 0) 0)) { + batch.size = 0; + } + } else if (inputColVector1.isRepeating) { + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (inputColVector1.compareTo(0, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (inputColVector1.compareTo(0, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector2.isRepeating) { + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (inputColVector1.compareTo(i, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (inputColVector1.compareTo(i, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + + // handle case where only input 2 has nulls + } else if (inputColVector1.noNulls) { + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + if (nullPos2[0] || + !(inputColVector1.compareTo(0, inputColVector2, 0) 0)) { + batch.size = 0; + } + } else if (inputColVector1.isRepeating) { + + // no need to check for nulls in input 1 + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos2[i]) { + if (inputColVector1.compareTo(0, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos2[i]) { + if (inputColVector1.compareTo(0, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector2.isRepeating) { + if (nullPos2[0]) { + + // no values will qualify because every comparison will be with NULL + batch.size = 0; + return; + } + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { // neither input is repeating + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos2[i]) { + if (inputColVector1.compareTo(i, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos2[i]) { + if (inputColVector1.compareTo(i, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + + // handle case where only input 1 has nulls + } else if (inputColVector2.noNulls) { + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + if (nullPos1[0] || + !(inputColVector1.compareTo(0, inputColVector2, 0) 0)) { + batch.size = 0; + return; + } + } else if (inputColVector1.isRepeating) { + if (nullPos1[0]) { + + // if repeating value is null then every comparison will fail so nothing qualifies + batch.size = 0; + return; + } + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (inputColVector1.compareTo(0, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (inputColVector1.compareTo(0, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector2.isRepeating) { + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i]) { + if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos1[i]) { + if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { // neither input is repeating + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i]) { + if (inputColVector1.compareTo(i, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos1[i]) { + if (inputColVector1.compareTo(i, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + + // handle case where both inputs have nulls + } else { + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + if (nullPos1[0] || nullPos2[0] || + !(inputColVector1.compareTo(0, inputColVector2, 0) 0)) { + batch.size = 0; + } + } else if (inputColVector1.isRepeating) { + if (nullPos1[0]) { + batch.size = 0; + return; + } + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos2[i]) { + if (inputColVector1.compareTo(0, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos2[i]) { + if (inputColVector1.compareTo(0, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else if (inputColVector2.isRepeating) { + if (nullPos2[0]) { + batch.size = 0; + return; + } + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i]) { + if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos1[i]) { + if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { // neither input is repeating + if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos1[i] && !nullPos2[i]) { + if (inputColVector1.compareTo(i, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos1[i] && !nullPos2[i]) { + if (inputColVector1.compareTo(i, inputColVector2, i) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < batch.size) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + } + } + + @Override + public String getOutputType() { + return "boolean"; + } + + @Override + public int getOutputColumn() { + return -1; } @Override @@ -43,8 +442,8 @@ public class extends { VectorExpressionDescriptor.Mode.FILTER) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("timestamp"), - VectorExpressionDescriptor.ArgumentType.getType("timestamp")) + VectorExpressionDescriptor.ArgumentType.getType(""), + VectorExpressionDescriptor.ArgumentType.getType("")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumnBase.txt deleted file mode 100644 index b5a7a7a..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampColumnBase.txt +++ /dev/null @@ -1,429 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; - -/** - * Generated from template FilterTimestampColumnCompareColumn.txt, which covers binary comparison - * filter expressions between two columns. Output is not produced in a separate column. - * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum1; - private int colNum2; - - public (int colNum1, int colNum2) { - this.colNum1 = colNum1; - this.colNum2 = colNum2; - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1]; - TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2]; - int[] sel = batch.selected; - boolean[] nullPos1 = inputColVector1.isNull; - boolean[] nullPos2 = inputColVector2.isNull; - int n = batch.size; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - // handle case where neither input has nulls - if (inputColVector1.noNulls && inputColVector2.noNulls) { - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - - /* Either all must remain selected or all will be eliminated. - * Repeating property will not change. - */ - if (!(inputColVector1.compareTo(0, inputColVector2, 0) 0)) { - batch.size = 0; - } - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (inputColVector1.compareTo(0, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (inputColVector1.compareTo(0, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (inputColVector1.compareTo(i, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (inputColVector1.compareTo(i, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - - // handle case where only input 2 has nulls - } else if (inputColVector1.noNulls) { - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - if (nullPos2[0] || - !(inputColVector1.compareTo(0, inputColVector2, 0) 0)) { - batch.size = 0; - } - } else if (inputColVector1.isRepeating) { - - // no need to check for nulls in input 1 - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos2[i]) { - if (inputColVector1.compareTo(0, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos2[i]) { - if (inputColVector1.compareTo(0, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector2.isRepeating) { - if (nullPos2[0]) { - - // no values will qualify because every comparison will be with NULL - batch.size = 0; - return; - } - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { // neither input is repeating - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos2[i]) { - if (inputColVector1.compareTo(i, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos2[i]) { - if (inputColVector1.compareTo(i, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - - // handle case where only input 1 has nulls - } else if (inputColVector2.noNulls) { - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - if (nullPos1[0] || - !(inputColVector1.compareTo(0, inputColVector2, 0) 0)) { - batch.size = 0; - return; - } - } else if (inputColVector1.isRepeating) { - if (nullPos1[0]) { - - // if repeating value is null then every comparison will fail so nothing qualifies - batch.size = 0; - return; - } - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (inputColVector1.compareTo(0, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (inputColVector1.compareTo(0, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i]) { - if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos1[i]) { - if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { // neither input is repeating - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i]) { - if (inputColVector1.compareTo(i, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos1[i]) { - if (inputColVector1.compareTo(i, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - - // handle case where both inputs have nulls - } else { - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - if (nullPos1[0] || nullPos2[0] || - !(inputColVector1.compareTo(0, inputColVector2, 0) 0)) { - batch.size = 0; - } - } else if (inputColVector1.isRepeating) { - if (nullPos1[0]) { - batch.size = 0; - return; - } - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos2[i]) { - if (inputColVector1.compareTo(0, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos2[i]) { - if (inputColVector1.compareTo(0, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else if (inputColVector2.isRepeating) { - if (nullPos2[0]) { - batch.size = 0; - return; - } - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i]) { - if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos1[i]) { - if (inputColVector1.compareTo(i, inputColVector2, 0) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { // neither input is repeating - if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos1[i] && !nullPos2[i]) { - if (inputColVector1.compareTo(i, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos1[i] && !nullPos2[i]) { - if (inputColVector1.compareTo(i, inputColVector2, i) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < batch.size) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - } - } - - @Override - public String getOutputType() { - return "boolean"; - } - - @Override - public int getOutputColumn() { - return -1; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt index f744d9b..bab8508 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalar.txt @@ -20,24 +20,130 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; /** - * Generated from template FilterTimestampColumnCompareScalar.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type, however output is not - * produced in a separate column. + * Generated from template FilterColumnCompareScalar.txt, which covers binary comparison + * expressions between a column and a scalar, however output is not produced in a separate column. * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering. */ -public class extends { +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private value; - public (int colNum, Timestamp value) { - super(colNum, new PisaTimestamp(value)); + public (int colNum, value) { + this.colNum = colNum; + this.value = value; } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type . + inputColVector1 = () batch.cols[colNum]; + + int[] sel = batch.selected; + boolean[] nullPos = inputColVector1.isNull; + int n = batch.size; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector1.noNulls) { + if (inputColVector1.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + if (!(inputColVector1.compareTo(0, value) 0)) { + //Entire batch is filtered out. + batch.size = 0; + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (inputColVector1.compareTo(i, value) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (inputColVector1.compareTo(i, value) 0) { + sel[newSize++] = i; + } + } + if (newSize < n) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { + if (inputColVector1.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + if (!nullPos[0]) { + if (!(inputColVector1.compareTo(0, value) 0)) { + //Entire batch is filtered out. + batch.size = 0; + } + } else { + batch.size = 0; + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + if (inputColVector1.compareTo(i, value) 0) { + sel[newSize++] = i; + } + } + } + //Change the selected vector + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos[i]) { + if (inputColVector1.compareTo(i, value) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < n) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; } @Override @@ -47,8 +153,8 @@ public class extends { VectorExpressionDescriptor.Mode.FILTER) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("timestamp"), - VectorExpressionDescriptor.ArgumentType.getType("timestamp")) + VectorExpressionDescriptor.ArgumentType.getType(""), + VectorExpressionDescriptor.ArgumentType.getType("")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalarBase.txt ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalarBase.txt deleted file mode 100644 index c84b4bf..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampColumnCompareTimestampScalarBase.txt +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - -/** - * Generated from template FilterColumnCompareScalar.txt, which covers binary comparison - * expressions between a column and a scalar, however output is not produced in a separate column. - * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - - public (int colNum, PisaTimestamp value) { - this.colNum = colNum; - this.value = value; - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - TimestampColumnVector inputColVector = (TimestampColumnVector) batch.cols[colNum]; - int[] sel = batch.selected; - boolean[] nullPos = inputColVector.isNull; - int n = batch.size; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector.noNulls) { - if (inputColVector.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - if (!(inputColVector.compareTo(0, value) 0)) { - //Entire batch is filtered out. - batch.size = 0; - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (inputColVector.compareTo(i, value) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (inputColVector.compareTo(i, value) 0) { - sel[newSize++] = i; - } - } - if (newSize < n) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { - if (inputColVector.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - if (!nullPos[0]) { - if (!(inputColVector.compareTo(0, value) 0)) { - //Entire batch is filtered out. - batch.size = 0; - } - } else { - batch.size = 0; - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j=0; j != n; j++) { - int i = sel[j]; - if (!nullPos[i]) { - if (inputColVector.compareTo(i, value) 0) { - sel[newSize++] = i; - } - } - } - //Change the selected vector - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos[i]) { - if (inputColVector.compareTo(i, value) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < n) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - } - - @Override - public int getOutputColumn() { - return -1; - } - - @Override - public String getOutputType() { - return "boolean"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareLongDoubleColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareLongDoubleColumn.txt index c3cd3b4..5e418de 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareLongDoubleColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareLongDoubleColumn.txt @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -36,7 +35,7 @@ public class extends { private static final long serialVersionUID = 1L; public (Timestamp value, int colNum) { - super(new PisaTimestamp(value).(), colNum); + super(TimestampColumnVector.(value), colNum); } public () { diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt index 05ab310..ff5d11e 100644 --- ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumn.txt @@ -20,24 +20,132 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.common.type.HiveDecimal; /** - * Generated from template FilterTimestampScalarCompareTimestampColumn.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type, however output is not - * produced in a separate column. - * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering. + * This is a generated class to evaluate a comparison on a vector of timestamp + * values. */ -public class extends { +public class extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private int colNum; + private value; - public (Timestamp value, int colNum) { - super(new PisaTimestamp(value), colNum); + public ( value, int colNum) { + this.colNum = colNum; + this.value = value; } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + if (childExpressions != null) { + super.evaluateChildren(batch); + } + // Input #2 is type . + inputColVector2 = () batch.cols[colNum]; + + int[] sel = batch.selected; + boolean[] nullPos = inputColVector2.isNull; + int n = batch.size; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector2.noNulls) { + if (inputColVector2.isRepeating) { + + // All must be selected otherwise size would be zero. Repeating property will not change. + if (!(inputColVector2.compareTo(value, 0) 0)) { + + // Entire batch is filtered out. + batch.size = 0; + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (inputColVector2.compareTo(value, i) 0) { + sel[newSize++] = i; + } + } + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (inputColVector2.compareTo(value, i) 0) { + sel[newSize++] = i; + } + } + if (newSize < n) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } else { + if (inputColVector2.isRepeating) { + + // All must be selected otherwise size would be zero. Repeating property will not change. + if (!nullPos[0]) { + if (!(inputColVector2.compareTo(value, 0) 0)) { + + // Entire batch is filtered out. + batch.size = 0; + } + } else { + batch.size = 0; + } + } else if (batch.selectedInUse) { + int newSize = 0; + for(int j = 0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + if (inputColVector2.compareTo(value, i) 0) { + sel[newSize++] = i; + } + } + } + + // Change the selected vector + batch.size = newSize; + } else { + int newSize = 0; + for(int i = 0; i != n; i++) { + if (!nullPos[i]) { + if (inputColVector2.compareTo(value, i) 0) { + sel[newSize++] = i; + } + } + } + if (newSize < n) { + batch.size = newSize; + batch.selectedInUse = true; + } + } + } + } + + @Override + public int getOutputColumn() { + return -1; + } + + @Override + public String getOutputType() { + return "boolean"; } @Override @@ -47,8 +155,8 @@ public class extends { VectorExpressionDescriptor.Mode.FILTER) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("timestamp"), - VectorExpressionDescriptor.ArgumentType.getType("timestamp")) + VectorExpressionDescriptor.ArgumentType.getType(""), + VectorExpressionDescriptor.ArgumentType.getType("")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.SCALAR, VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); diff --git ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumnBase.txt deleted file mode 100644 index 608faef..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/FilterTimestampScalarCompareTimestampColumnBase.txt +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; -import org.apache.hadoop.hive.common.type.HiveDecimal; - -/** - * This is a generated class to evaluate a comparison on a vector of timestamp - * values. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - - public (PisaTimestamp value, int colNum) { - this.colNum = colNum; - this.value = value; - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - if (childExpressions != null) { - super.evaluateChildren(batch); - } - TimestampColumnVector inputColVector = (TimestampColumnVector) batch.cols[colNum]; - int[] sel = batch.selected; - boolean[] nullPos = inputColVector.isNull; - int n = batch.size; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector.noNulls) { - if (inputColVector.isRepeating) { - - // All must be selected otherwise size would be zero. Repeating property will not change. - if (!(inputColVector.compareTo(value, 0) 0)) { - - // Entire batch is filtered out. - batch.size = 0; - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (inputColVector.compareTo(value, i) 0) { - sel[newSize++] = i; - } - } - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (inputColVector.compareTo(value, i) 0) { - sel[newSize++] = i; - } - } - if (newSize < n) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } else { - if (inputColVector.isRepeating) { - - // All must be selected otherwise size would be zero. Repeating property will not change. - if (!nullPos[0]) { - if (!(inputColVector.compareTo(value, 0) 0)) { - - // Entire batch is filtered out. - batch.size = 0; - } - } else { - batch.size = 0; - } - } else if (batch.selectedInUse) { - int newSize = 0; - for(int j = 0; j != n; j++) { - int i = sel[j]; - if (!nullPos[i]) { - if (inputColVector.compareTo(value, i) 0) { - sel[newSize++] = i; - } - } - } - - // Change the selected vector - batch.size = newSize; - } else { - int newSize = 0; - for(int i = 0; i != n; i++) { - if (!nullPos[i]) { - if (inputColVector.compareTo(value, i) 0) { - sel[newSize++] = i; - } - } - } - if (newSize < n) { - batch.size = newSize; - batch.selectedInUse = true; - } - } - } - } - - @Override - public int getOutputColumn() { - return -1; - } - - @Override - public String getOutputType() { - return "boolean"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeColumn.txt ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeColumn.txt deleted file mode 100644 index bf62b78..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeColumn.txt +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - - -/** - * Generated from template IntervalDayTimeColumnCompareColumn.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type. The boolean output - * is stored in a separate boolean column. - */ -public class extends { - - private static final long serialVersionUID = 1L; - - public (int colNum1, int colNum2, int outputColumn) { - super(colNum1, colNum2, outputColumn); - } - - public () { - super(); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"), - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeScalar.txt ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeScalar.txt deleted file mode 100644 index 1abb4a3..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeColumnCompareIntervalDayTimeScalar.txt +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - - -/** - * Generated from template IntervalDayTimeColumnCompareScalar.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type. The boolean output - * is stored in a separate boolean column. - */ -public class extends { - - private static final long serialVersionUID = 1L; - - public (int colNum, HiveIntervalDayTime value, int outputColumn) { - super(colNum, value.pisaTimestampUpdate(new PisaTimestamp()), outputColumn); - } - - public () { - super(); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"), - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.COLUMN, - VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeScalarCompareIntervalDayTimeColumn.txt ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeScalarCompareIntervalDayTimeColumn.txt deleted file mode 100644 index 26762ff..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/IntervalDayTimeScalarCompareIntervalDayTimeColumn.txt +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - - -/** - * Generated from template IntervalDayTimeColumnCompareScalar.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type. The boolean output - * is stored in a separate boolean column. - */ -public class extends { - - private static final long serialVersionUID = 1L; - - public (HiveIntervalDayTime value, int colNum, int outputColumn) { - super(value.pisaTimestampUpdate(new PisaTimestamp()), colNum, outputColumn); - } - - public () { - super(); - } - - @Override - public VectorExpressionDescriptor.Descriptor getDescriptor() { - return (new VectorExpressionDescriptor.Builder()) - .setMode( - VectorExpressionDescriptor.Mode.PROJECTION) - .setNumArguments(2) - .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time"), - VectorExpressionDescriptor.ArgumentType.getType("interval_day_time")) - .setInputExpressionTypes( - VectorExpressionDescriptor.InputExpressionType.SCALAR, - VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateColumn.txt ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateColumn.txt index 7ae84b7..8e3a419 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateColumn.txt @@ -18,12 +18,15 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** * Generated from template DateColumnArithmeticIntervalYearMonthColumn.txt, which covers binary arithmetic @@ -36,12 +39,18 @@ public class extends VectorExpression { private int colNum1; private int colNum2; private int outputColumn; + private HiveIntervalYearMonth scratchIntervalYearMonth1; + private Date scratchDate2; + private Date outputDate; private DateTimeMath dtm = new DateTimeMath(); public (int colNum1, int colNum2, int outputColumn) { this.colNum1 = colNum1; this.colNum2 = colNum2; this.outputColumn = outputColumn; + scratchIntervalYearMonth1 = new HiveIntervalYearMonth(); + scratchDate2 = new Date(0); + outputDate = new Date(0); } public () { @@ -54,10 +63,10 @@ public class extends VectorExpression { super.evaluateChildren(batch); } - // Input #1 is type interval_year_month (months). + // Input #1 is type interval_year_month. LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - // Input #2 is type date (epochDays). + // Input #2 is type date. LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; // Output is type date. @@ -89,40 +98,64 @@ public class extends VectorExpression { * conditional checks in the inner loop. */ if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = dtm.addMonthsToDays(vector2[0], (int) vector1[0]); + scratchIntervalYearMonth1.set((int) vector1[0]); + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[0])); + dtm.( + scratchIntervalYearMonth1, scratchDate2, outputDate); + outputVector[0] = DateWritable.dateToDays(outputDate); } else if (inputColVector1.isRepeating) { - long value1 = vector1[0]; + scratchIntervalYearMonth1.set((int) vector1[0]); if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector2[i], (int) value1); + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + scratchIntervalYearMonth1, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector2[i], (int) value1); + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + scratchIntervalYearMonth1, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else if (inputColVector2.isRepeating) { - long value2 = vector2[0]; + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[0])); if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(value2, (int) vector1[i]); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(value2, (int) vector1[i]); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector2[i], (int) vector1[i]); + scratchIntervalYearMonth1.set((int) vector1[i]); + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + scratchIntervalYearMonth1, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector2[i], (int) vector1[i]); + scratchIntervalYearMonth1.set((int) vector1[i]); + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + scratchIntervalYearMonth1, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } diff --git ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateScalar.txt ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateScalar.txt index 2f2522d..ad65d52 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticDateScalar.txt @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -25,6 +27,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** * Generated from template DateColumnArithmeticIntervalYearMonthScalar.txt, which covers binary arithmetic @@ -35,14 +38,18 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; private int colNum; - private long value; + private Date value; private int outputColumn; + private HiveIntervalYearMonth scratchIntervalYearMonth1; + private Date outputDate; private DateTimeMath dtm = new DateTimeMath(); public (int colNum, long value, int outputColumn) { this.colNum = colNum; - this.value = value; + this.value = new Date(DateWritable.daysToMillis((int) value)); this.outputColumn = outputColumn; + scratchIntervalYearMonth1 = new HiveIntervalYearMonth(); + outputDate = new Date(0); } public () { @@ -56,18 +63,18 @@ public class extends VectorExpression { } // Input #1 is type interval_year_mont (epochMonths). - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; + LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum]; // Output is type date. LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; + boolean[] inputIsNull = inputColVector1.isNull; boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - outputColVector.isRepeating = inputColVector.isRepeating; + outputColVector.noNulls = inputColVector1.noNulls; + outputColVector.isRepeating = inputColVector1.isRepeating; int n = batch.size; - long[] vector = inputColVector.vector; + long[] vector1 = inputColVector1.vector; long[] outputVector = outputColVector.vector; // return immediately if batch is empty @@ -75,32 +82,46 @@ public class extends VectorExpression { return; } - if (inputColVector.isRepeating) { - outputVector[0] = dtm.addMonthsToDays(value, (int) vector[0]); - + if (inputColVector1.isRepeating) { + scratchIntervalYearMonth1.set((int) vector1[0]); + dtm.( + scratchIntervalYearMonth1, value, outputDate); + outputVector[0] = DateWritable.dateToDays(outputDate); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector.noNulls) { + } else if (inputColVector1.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(value, (int) vector[i]); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(value, (int) vector[i]); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else /* there are nulls */ { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(value, (int) vector[i]); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(value, (int) vector[i]); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, value, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampColumn.txt index b3da89f..858c3d7 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampColumn.txt @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -37,14 +39,14 @@ public class extends VectorExpression { private int colNum1; private int colNum2; private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; + private HiveIntervalYearMonth scratchIntervalYearMonth1; private DateTimeMath dtm = new DateTimeMath(); public (int colNum1, int colNum2, int outputColumn) { this.colNum1 = colNum1; this.colNum2 = colNum2; this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); + scratchIntervalYearMonth1 = new HiveIntervalYearMonth(); } public () { @@ -57,10 +59,10 @@ public class extends VectorExpression { super.evaluateChildren(batch); } - // Input #1 is type Interval_Year_Month (months). + // Input #1 is type Interval_Year_Month. LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; - // Input #2 is type Timestamp (PisaTimestamp). + // Input #2 is type Timestamp. TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2]; // Output is type Timestamp. @@ -91,54 +93,59 @@ public class extends VectorExpression { * conditional checks in the inner loop. */ if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputColVector.set(0, - dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(0), (int) vector1[0], - scratchPisaTimestamp)); + scratchIntervalYearMonth1.set((int) vector1[0]); + dtm.( + scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(0), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(0); } else if (inputColVector1.isRepeating) { - long value1 = vector1[0]; + scratchIntervalYearMonth1.set((int) vector1[0]); if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(i), (int) value1, - scratchPisaTimestamp)); + dtm.( + scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(i), (int) value1, - scratchPisaTimestamp)); + dtm.( + scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } else if (inputColVector2.isRepeating) { - PisaTimestamp value2 = inputColVector2.asScratchPisaTimestamp(0); + Timestamp value2 = inputColVector2.asScratchTimestamp(0); if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value2, (int) vector1[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value2, (int) vector1[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } else { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(i), (int) vector1[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector2.asScratchPisaTimestamp(i), (int) vector1[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } diff --git ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampScalar.txt ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampScalar.txt index 81f2a77..66fffd2 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthColumnArithmeticTimestampScalar.txt @@ -20,7 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -38,16 +38,16 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; private int colNum; - private PisaTimestamp value; + private Timestamp value; private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; + private HiveIntervalYearMonth scratchIntervalYearMonth1; private DateTimeMath dtm = new DateTimeMath(); public (int colNum, Timestamp value, int outputColumn) { this.colNum = colNum; - this.value = new PisaTimestamp(value); + this.value = value; this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); + scratchIntervalYearMonth1 = new HiveIntervalYearMonth(); } public () { @@ -60,7 +60,7 @@ public class extends VectorExpression { super.evaluateChildren(batch); } - // Input #1 is type interval_year_month (epochMonths). + // Input #1 is type interval_year_month. LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum]; // Output is type Timestamp. @@ -81,41 +81,45 @@ public class extends VectorExpression { } if (inputColVector1.isRepeating) { - outputColVector.set(0, - dtm.addMonthsToPisaTimestamp(value, (int) vector1[0], - scratchPisaTimestamp)); - + scratchIntervalYearMonth1.set((int) vector1[0]); + dtm.( + scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; } else if (inputColVector1.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value, (int) vector1[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value, (int) vector1[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } else /* there are nulls */ { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value, (int) vector1[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value, (int) vector1[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth1.set((int) vector1[i]); + dtm.( + scratchIntervalYearMonth1, value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticDateColumn.txt ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticDateColumn.txt index 3f4f05f..ddde913 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticDateColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticDateColumn.txt @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Date; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -33,6 +35,7 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** * Generated from template DateTimeScalarArithmeticIntervalYearMonthColumn.txt. @@ -44,14 +47,18 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; private int colNum; - private long value; + private HiveIntervalYearMonth value; private int outputColumn; + private Date scratchDate2; + private Date outputDate; private DateTimeMath dtm = new DateTimeMath(); public (long value, int colNum, int outputColumn) { this.colNum = colNum; - this.value = value; + this.value = new HiveIntervalYearMonth((int) value); this.outputColumn = outputColumn; + scratchDate2 = new Date(0); + outputDate = new Date(0); } public () { @@ -70,18 +77,18 @@ public class extends VectorExpression { } // Input #2 is type date. - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; + LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum]; // Output is type Date. LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; + boolean[] inputIsNull = inputColVector2.isNull; boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - outputColVector.isRepeating = inputColVector.isRepeating; + outputColVector.noNulls = inputColVector2.noNulls; + outputColVector.isRepeating = inputColVector2.isRepeating; int n = batch.size; - long[] vector = inputColVector.vector; + long[] vector2 = inputColVector2.vector; long[] outputVector = outputColVector.vector; // return immediately if batch is empty @@ -89,32 +96,46 @@ public class extends VectorExpression { return; } - if (inputColVector.isRepeating) { - outputVector[0] = dtm.addMonthsToDays(vector[0], (int) value); - + if (inputColVector2.isRepeating) { + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[0])); + dtm.( + value, scratchDate2, outputDate); + outputVector[0] = DateWritable.dateToDays(outputDate); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector.noNulls) { + } else if (inputColVector2.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector[i], (int) value); + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector[i], (int) value); + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } } } else { /* there are nulls */ if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = dtm.addMonthsToDays(vector[i], (int) value); + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputVector[i] = dtm.addMonthsToDays(vector[i], (int) value); + scratchDate2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value, scratchDate2, outputDate); + outputVector[i] = DateWritable.dateToDays(outputDate); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticTimestampColumn.txt index 47d611e..cbb7021 100644 --- ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/IntervalYearMonthScalarArithmeticTimestampColumn.txt @@ -18,11 +18,13 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.common.type.PisaTimestamp; /* * Because of the templatized nature of the code, either or both * of these ColumnVector imports may be needed. Listing both of them @@ -44,16 +46,14 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; private int colNum; - private long value; + private HiveIntervalYearMonth value; private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; private DateTimeMath dtm = new DateTimeMath(); public (long value, int colNum, int outputColumn) { this.colNum = colNum; - this.value = value; + this.value = new HiveIntervalYearMonth((int) value); this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); } public () { @@ -72,16 +72,16 @@ public class extends VectorExpression { } // Input #2 is type timestamp. - TimestampColumnVector inputColVector = (TimestampColumnVector) batch.cols[colNum]; + TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum]; - // Output is type Timestamp. + // Output is type Timestamp. TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; + boolean[] inputIsNull = inputColVector2.isNull; boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - outputColVector.isRepeating = inputColVector.isRepeating; + outputColVector.noNulls = inputColVector2.noNulls; + outputColVector.isRepeating = inputColVector2.isRepeating; int n = batch.size; // return immediately if batch is empty @@ -89,42 +89,41 @@ public class extends VectorExpression { return; } - if (inputColVector.isRepeating) { - outputColVector.set(0, - dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(0), (int) value, - scratchPisaTimestamp)); - + if (inputColVector2.isRepeating) { + dtm.( + value, inputColVector2.asScratchTimestamp(0), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector.noNulls) { + } else if (inputColVector2.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(i), (int) value, - scratchPisaTimestamp)); + dtm.( + value, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(i), (int) value, - scratchPisaTimestamp)); + dtm.( + value, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } else { /* there are nulls */ if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(i), (int) value, - scratchPisaTimestamp)); + dtm.( + value, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector.asScratchPisaTimestamp(i), (int) value, - scratchPisaTimestamp)); + dtm.( + value, inputColVector2.asScratchTimestamp(i), outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampColumn.txt index e804e2a..9ccfaac 100644 --- ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampColumn.txt @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; diff --git ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampScalar.txt ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampScalar.txt index 90720ba..c7d8c65 100644 --- ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/LongDoubleColumnCompareTimestampScalar.txt @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -41,7 +40,7 @@ public class extends VectorExpression { public (int colNum, Timestamp value, int outputColumn) { this.colNum = colNum; - this.value = new PisaTimestamp(value).(); + this.value = TimestampColumnVector.(value); this.outputColumn = outputColumn; } diff --git ql/src/gen/vectorization/ExpressionTemplates/LongDoubleScalarCompareTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/LongDoubleScalarCompareTimestampColumn.txt index 7d3856a..d47bc10 100644 --- ql/src/gen/vectorization/ExpressionTemplates/LongDoubleScalarCompareTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/LongDoubleScalarCompareTimestampColumn.txt @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumn.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumn.txt index b086a88..27e083d 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumn.txt @@ -18,28 +18,156 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** * Generated from template TimestampColumnArithmeticDateColumn.txt, which covers binary arithmetic * expressions between columns. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; + private int colNum1; + private int colNum2; + private int outputColumn; + private Timestamp scratchTimestamp2; + private DateTimeMath dtm = new DateTimeMath(); + public (int colNum1, int colNum2, int outputColumn) { - super(colNum1, colNum2, outputColumn); + this.colNum1 = colNum1; + this.colNum2 = colNum2; + this.outputColumn = outputColumn; + scratchTimestamp2 = new Timestamp(0); } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type . + inputColVector1 = () batch.cols[colNum1]; + + // Input #2 is type date (days). For the math we convert it to a timestamp. + LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; + + // Output is type . + outputColVector = () batch.cols[outputColumn]; + + int[] sel = batch.selected; + int n = batch.size; + + long[] vector2 = inputColVector2.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + outputColVector.isRepeating = + inputColVector1.isRepeating && inputColVector2.isRepeating + || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] + || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; + + // Handle nulls first + NullUtil.propagateNullsColCol( + inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); + + /* Disregard nulls for processing. In other words, + * the arithmetic operation is performed even if one or + * more inputs are null. This is to improve speed by avoiding + * conditional checks in the inner loop. + */ + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0])); + dtm.( + inputColVector1.asScratch(0), scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(0); + } else if (inputColVector1.isRepeating) { + value1 = inputColVector1.asScratch(0); + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value1, scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value1, scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else if (inputColVector2.isRepeating) { + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0])); + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + inputColVector1.asScratch(i), scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + inputColVector1.asScratch(i), scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + inputColVector1.asScratch(i), scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + inputColVector1.asScratch(i), scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } + + /* For the case when the output can have null values, follow + * the convention that the data values must be 1 for long and + * NaN for double. This is to prevent possible later zero-divide errors + * in complex arithmetic expressions like col2 / (col1 - 1) + * in the case when some col1 entries are null. + */ + NullUtil.setNullDataEntries(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "interval_day_time"; } @Override @@ -50,7 +178,7 @@ public class extends { .setNumArguments(2) .setArgumentTypes( VectorExpressionDescriptor.ArgumentType.getType(""), - VectorExpressionDescriptor.ArgumentType.getType("")) + VectorExpressionDescriptor.ArgumentType.getType("date")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumnBase.txt deleted file mode 100644 index 7f5496c..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateColumnBase.txt +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.serde2.io.DateWritable; - -/** - * Generated from template TimestampColumnArithmeticDateColumnBase.txt, which covers binary arithmetic - * expressions between columns. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum1; - private int colNum2; - private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; - - public (int colNum1, int colNum2, int outputColumn) { - this.colNum1 = colNum1; - this.colNum2 = colNum2; - this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - // Input #1 is type timestamp (PisaTimestamp). - TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1]; - - // Input #2 is type date. - LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; - - // Output is type timestamp. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; - - int[] sel = batch.selected; - int n = batch.size; - - long[] vector2 = inputColVector2.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - outputColVector.isRepeating = - inputColVector1.isRepeating && inputColVector2.isRepeating - || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] - || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; - - // Handle nulls first - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(0), - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])), - 0); - } else if (inputColVector1.isRepeating) { - PisaTimestamp value1 = inputColVector1.asScratchPisaTimestamp(0); - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - value1, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - value1, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); - } - } - } else if (inputColVector2.isRepeating) { - PisaTimestamp value2 = - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])); - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), - value2, - i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), - value2, - i); - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesTimestamp(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "timestamp"; - } -} - diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalar.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalar.txt index b8404db..8b91a4a 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalar.txt @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -26,21 +28,107 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.util.DateTimeMath; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** * Generated from template TimestampColumnArithmeticDateScalar.txt, which covers binary arithmetic * expressions between a column and a scalar. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; + private int colNum; + private Timestamp value; + private int outputColumn; + private DateTimeMath dtm = new DateTimeMath(); + public (int colNum, long value, int outputColumn) { - super(colNum, value, outputColumn); + this.colNum = colNum; + this.value = new Timestamp(0); + this.value.setTime(DateWritable.daysToMillis((int) value)); + this.outputColumn = outputColumn; } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type . + inputColVector1 = () batch.cols[colNum]; + + // Output is type . + outputColVector = () batch.cols[outputColumn]; + + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector1.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector1.noNulls; + outputColVector.isRepeating = inputColVector1.isRepeating; + int n = batch.size; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector1.isRepeating) { + dtm.( + inputColVector1.asScratch(0), value, outputColVector.getScratch()); + outputColVector.setFromScratch(0); + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + } else if (inputColVector1.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + inputColVector1.asScratch(i), value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + inputColVector1.asScratch(i), value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + inputColVector1.asScratch(i), value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + inputColVector1.asScratch(i), value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + } + + NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; } @Override @@ -51,7 +139,7 @@ public class extends { .setNumArguments(2) .setArgumentTypes( VectorExpressionDescriptor.ArgumentType.getType(""), - VectorExpressionDescriptor.ArgumentType.getType("")) + VectorExpressionDescriptor.ArgumentType.getType("date")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalarBase.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalarBase.txt deleted file mode 100644 index c2ddd67..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticDateScalarBase.txt +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.serde2.io.DateWritable; - -/** - * Generated from template TimestampColumnArithmeticDateScalarBase.txt, which covers binary arithmetic - * expressions between a column and a scalar. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; - - public (int colNum, long value, int outputColumn) { - this.colNum = colNum; - this.value = new PisaTimestamp().updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) value)); - this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - // Input #1 is type Timestamp (PisaTimestamp). - TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum]; - - // Output is type Timestamp. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; - - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector1.isNull; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector1.noNulls; - outputColVector.isRepeating = inputColVector1.isRepeating; - int n = batch.size; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector1.isRepeating) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(0), value, 0); - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector1.noNulls) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), value, i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), value, i); - } - } - } else /* there are nulls */ { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), value, i); - outputIsNull[i] = inputIsNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), value, i); - } - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - } - } - - NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "timestamp"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthColumn.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthColumn.txt index 2f33920..4ac2174 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthColumn.txt @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -37,14 +39,14 @@ public class extends VectorExpression { private int colNum1; private int colNum2; private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; + private HiveIntervalYearMonth scratchIntervalYearMonth2; private DateTimeMath dtm = new DateTimeMath(); public (int colNum1, int colNum2, int outputColumn) { this.colNum1 = colNum1; this.colNum2 = colNum2; this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); + scratchIntervalYearMonth2 = new HiveIntervalYearMonth(); } public () { @@ -57,7 +59,7 @@ public class extends VectorExpression { super.evaluateChildren(batch); } - // Input #1 is type Timestamp (PisaTimestamp). + // Input #1 is type Timestamp. TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1]; // Input #2 is type Interval_Year_Month (months). @@ -91,52 +93,59 @@ public class extends VectorExpression { * conditional checks in the inner loop. */ if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputColVector.set(0, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(0), (int) vector2[0], - scratchPisaTimestamp)); + scratchIntervalYearMonth2.set((int) vector2[0]); + dtm.( + inputColVector1.asScratchTimestamp(0), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(0); } else if (inputColVector1.isRepeating) { + Timestamp value1 = inputColVector1.asScratchTimestamp(0); if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(0), (int) vector2[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value1, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(0), (int) vector2[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value1, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } else if (inputColVector2.isRepeating) { + scratchIntervalYearMonth2.set((int) vector2[0]); if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), (int) vector2[0], - scratchPisaTimestamp)); + dtm.( + inputColVector1.asScratchTimestamp(i), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), (int) vector2[0], - scratchPisaTimestamp)); + dtm.( + inputColVector1.asScratchTimestamp(i), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } else { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), (int) vector2[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + inputColVector1.asScratchTimestamp(i), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), (int) vector2[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + inputColVector1.asScratchTimestamp(i), scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthScalar.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthScalar.txt index 9f5c24e..9382aca 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticIntervalYearMonthScalar.txt @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -36,16 +36,14 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; private int colNum; - private long value; + private HiveIntervalYearMonth value; private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; private DateTimeMath dtm = new DateTimeMath(); public (int colNum, long value, int outputColumn) { this.colNum = colNum; - this.value = value; + this.value = new HiveIntervalYearMonth((int) value); this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); } public () { @@ -58,7 +56,7 @@ public class extends VectorExpression { super.evaluateChildren(batch); } - // Input #1 is type Timestamp (PisaTimestamp). + // Input #1 is type Timestamp. TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum]; // Output is type Timestamp. @@ -77,41 +75,40 @@ public class extends VectorExpression { } if (inputColVector1.isRepeating) { - outputColVector.set(0, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(0), (int) value, - scratchPisaTimestamp)); - + dtm.( + inputColVector1.asScratchTimestamp(0), value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; } else if (inputColVector1.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), (int) value, - scratchPisaTimestamp)); + dtm.( + inputColVector1.asScratchTimestamp(i), value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), (int) value, - scratchPisaTimestamp)); + dtm.( + inputColVector1.asScratchTimestamp(i), value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } else /* there are nulls */ { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), (int) value, - scratchPisaTimestamp)); + dtm.( + inputColVector1.asScratchTimestamp(i), value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(inputColVector1.asScratchPisaTimestamp(i), (int) value, - scratchPisaTimestamp)); + dtm.( + inputColVector1.asScratchTimestamp(i), value, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumn.txt index dfd45ab..5eaa450 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumn.txt @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -27,19 +29,135 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.util.DateTimeMath; /** - * Generated from template TimestampColumnArithmeticTimestampColumn.txt, which covers binary arithmetic + * Generated from template TimestampColumnArithmeticTimestampColumnBase.txt, which covers binary arithmetic * expressions between columns. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; + private int colNum1; + private int colNum2; + private int outputColumn; + private DateTimeMath dtm = new DateTimeMath(); + public (int colNum1, int colNum2, int outputColumn) { - super(colNum1, colNum2, outputColumn); + this.colNum1 = colNum1; + this.colNum2 = colNum2; + this.outputColumn = outputColumn; } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type . + inputColVector1 = () batch.cols[colNum1]; + + // Input #2 is type . + inputColVector2 = () batch.cols[colNum2]; + + // Output is type . + outputColVector = () batch.cols[outputColumn]; + + int[] sel = batch.selected; + int n = batch.size; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + outputColVector.isRepeating = + inputColVector1.isRepeating && inputColVector2.isRepeating + || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] + || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; + + // Handle nulls first + NullUtil.propagateNullsColCol( + inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); + + /* Disregard nulls for processing. In other words, + * the arithmetic operation is performed even if one or + * more inputs are null. This is to improve speed by avoiding + * conditional checks in the inner loop. + */ + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + dtm.( + inputColVector1.asScratch(0), inputColVector2.asScratch(0), outputColVector.getScratch()); + outputColVector.setFromScratch(0); + } else if (inputColVector1.isRepeating) { + value1 = inputColVector1.asScratch(0); + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + value1, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + value1, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else if (inputColVector2.isRepeating) { + value2 = inputColVector2.asScratch(0); + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + inputColVector1.asScratch(i), value2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + inputColVector1.asScratch(i), value2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + inputColVector1.asScratch(i), inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + inputColVector1.asScratch(i), inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } + + /* For the case when the output can have null values, follow + * the convention that the data values must be 1 for long and + * NaN for double. This is to prevent possible later zero-divide errors + * in complex arithmetic expressions like col2 / (col1 - 1) + * in the case when some col1 entries are null. + */ + NullUtil.setNullDataEntries(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; } @Override diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumnBase.txt deleted file mode 100644 index 0e52f6c..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampColumnBase.txt +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.util.DateTimeMath; - -/** - * Generated from template TimestampColumnArithmeticTimestampColumnBase.txt, which covers binary arithmetic - * expressions between columns. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum1; - private int colNum2; - private int outputColumn; - private DateTimeMath dtm = new DateTimeMath(); - - public (int colNum1, int colNum2, int outputColumn) { - this.colNum1 = colNum1; - this.colNum2 = colNum2; - this.outputColumn = outputColumn; - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - // Input #1 is type timestamp/interval_day_time (PisaTimestamp). - TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1]; - - // Input #2 is type timestamp/interval_day_time (PisaTimestamp). - TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2]; - - // Output is type timestamp/interval_day_time (PisaTimestamp). - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; - - int[] sel = batch.selected; - int n = batch.size; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - outputColVector.isRepeating = - inputColVector1.isRepeating && inputColVector2.isRepeating - || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] - || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; - - // Handle nulls first - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(0), inputColVector2.asScratchPisaTimestamp(0), 0); - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - inputColVector1.asScratchPisaTimestamp(0), inputColVector2.asScratchPisaTimestamp(i), i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(0), inputColVector2.asScratchPisaTimestamp(i), i); - } - } - } else if (inputColVector2.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), inputColVector2.asScratchPisaTimestamp(0), i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), inputColVector2.asScratchPisaTimestamp(0), i); - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), inputColVector2.asScratchPisaTimestamp(i), i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), inputColVector2.asScratchPisaTimestamp(i), i); - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesTimestamp(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "timestamp"; - } -} - diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalar.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalar.txt index f8004ff..c6c872f 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalar.txt @@ -19,10 +19,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hive.common.util.DateUtils; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -35,16 +33,100 @@ import org.apache.hadoop.hive.ql.util.DateTimeMath; * Generated from template TimestampColumnArithmeticTimestampScalar.txt, which covers binary arithmetic * expressions between a column and a scalar. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; - public (int colNum, value, int outputColumn) { - super(colNum, , outputColumn); + private int colNum; + private value; + private int outputColumn; + private DateTimeMath dtm = new DateTimeMath(); + + public (int colNum, value, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type . + inputColVector1 = () batch.cols[colNum]; + + // Output is type . + outputColVector = () batch.cols[outputColumn]; + + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector1.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector1.noNulls; + outputColVector.isRepeating = inputColVector1.isRepeating; + int n = batch.size; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector1.isRepeating) { + dtm.( + inputColVector1.asScratch(0), value, outputColVector.getScratch()); + outputColVector.setFromScratch(0); + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + } else if (inputColVector1.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + inputColVector1.asScratch(i), value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + inputColVector1.asScratch(i), value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + inputColVector1.asScratch(i), value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + inputColVector1.asScratch(i), value, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + } + + NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; } @Override diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalarBase.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalarBase.txt deleted file mode 100644 index a0de1b3..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnArithmeticTimestampScalarBase.txt +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.util.DateTimeMath; - -/** - * Generated from template TimestampColumnArithmeticTimestampScalarBase.txt, which covers binary arithmetic - * expressions between a column and a scalar. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - private int outputColumn; - private DateTimeMath dtm = new DateTimeMath(); - - public (int colNum, PisaTimestamp value, int outputColumn) { - this.colNum = colNum; - this.value = value; - this.outputColumn = outputColumn; - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - // Input #1 is type timestamp/interval_day_time (PisaTimestamp). - TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum]; - - // Output is type timestamp/interval_day_time. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; - - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector1.isNull; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector1.noNulls; - outputColVector.isRepeating = inputColVector1.isRepeating; - int n = batch.size; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector1.isRepeating) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(0), value, 0); - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector1.noNulls) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), value, i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), value, i); - } - } - } else /* there are nulls */ { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), value, i); - outputIsNull[i] = inputIsNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - inputColVector1.asScratchPisaTimestamp(i), value, i); - } - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - } - } - - NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "timestamp"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt index 43321644..e0ae206 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareLongDoubleScalar.txt @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt index fb82d5e..f9fc425 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumn.txt @@ -18,24 +18,128 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import java.sql.Timestamp; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; /** - * Generated from template TimestampColumnCompareTimestampColumn.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type. The boolean output - * is stored in a separate boolean column. + * Generated from template TimestampColumnCompareColumn.txt, which covers comparision + * expressions between timestamp columns. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; + private int colNum1; + private int colNum2; + private int outputColumn; + public (int colNum1, int colNum2, int outputColumn) { - super(colNum1, colNum2, outputColumn); + this.colNum1 = colNum1; + this.colNum2 = colNum2; + this.outputColumn = outputColumn; } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type . + inputColVector1 = () batch.cols[colNum1]; + + // Input #2 is type . + inputColVector2 = () batch.cols[colNum2]; + + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + int n = batch.size; + long[] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + outputColVector.isRepeating = + inputColVector1.isRepeating && inputColVector2.isRepeating + || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] + || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; + + // Handle nulls first + NullUtil.propagateNullsColCol( + inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); + + /* Disregard nulls for processing. In other words, + * the arithmetic operation is performed even if one or + * more inputs are null. This is to improve speed by avoiding + * conditional checks in the inner loop. + */ + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + outputVector[0] = inputColVector1.compareTo(0, inputColVector2.asScratch(0)) 0 ? 1 : 0; + } else if (inputColVector1.isRepeating) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = inputColVector1.compareTo(0, inputColVector2.asScratch(i)) 0 ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = inputColVector1.compareTo(0, inputColVector2.asScratch(i)) 0 ? 1 : 0; + } + } + } else if (inputColVector2.isRepeating) { + value2 = inputColVector2.asScratch(0); + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = inputColVector1.compareTo(i, value2) 0 ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = inputColVector1.compareTo(i, value2) 0 ? 1 : 0; + } + } + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = inputColVector1.compareTo(i, inputColVector2.asScratch(i)) 0 ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = inputColVector1.compareTo(i, inputColVector2.asScratch(i)) 0 ? 1 : 0; + } + } + } + + /* For the case when the output can have null values, follow + * the convention that the data values must be 1 for long and + * NaN for double. This is to prevent possible later zero-divide errors + * in complex arithmetic expressions like col2 / (col1 - 1) + * in the case when some col1 entries are null. + */ + NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "long"; } @Override @@ -45,8 +149,8 @@ public class extends { VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("timestamp"), - VectorExpressionDescriptor.ArgumentType.getType("timestamp")) + VectorExpressionDescriptor.ArgumentType.getType(""), + VectorExpressionDescriptor.ArgumentType.getType("")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumnBase.txt deleted file mode 100644 index 302be41..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampColumnBase.txt +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - -/** - * Generated from template TimestampColumnCompareColumn.txt, which covers comparision - * expressions between timestamp columns. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum1; - private int colNum2; - private int outputColumn; - - public (int colNum1, int colNum2, int outputColumn) { - this.colNum1 = colNum1; - this.colNum2 = colNum2; - this.outputColumn = outputColumn; - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum1]; - TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum2]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; - int[] sel = batch.selected; - int n = batch.size; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - outputColVector.isRepeating = - inputColVector1.isRepeating && inputColVector2.isRepeating - || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] - || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; - - // Handle nulls first - NullUtil.propagateNullsColCol( - inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); - - /* Disregard nulls for processing. In other words, - * the arithmetic operation is performed even if one or - * more inputs are null. This is to improve speed by avoiding - * conditional checks in the inner loop. - */ - if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputVector[0] = inputColVector1.compareTo(0, inputColVector2.asScratchPisaTimestamp(0)) 0 ? 1 : 0; - } else if (inputColVector1.isRepeating) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = inputColVector1.compareTo(0, inputColVector2.asScratchPisaTimestamp(i)) 0 ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = inputColVector1.compareTo(0, inputColVector2.asScratchPisaTimestamp(i)) 0 ? 1 : 0; - } - } - } else if (inputColVector2.isRepeating) { - PisaTimestamp value2 = inputColVector2.asScratchPisaTimestamp(0); - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = inputColVector1.compareTo(i, value2) 0 ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = inputColVector1.compareTo(i, value2) 0 ? 1 : 0; - } - } - } else { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputVector[i] = inputColVector1.compareTo(i, inputColVector2.asScratchPisaTimestamp(i)) 0 ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = inputColVector1.compareTo(i, inputColVector2.asScratchPisaTimestamp(i)) 0 ? 1 : 0; - } - } - } - - /* For the case when the output can have null values, follow - * the convention that the data values must be 1 for long and - * NaN for double. This is to prevent possible later zero-divide errors - * in complex arithmetic expressions like col2 / (col1 - 1) - * in the case when some col1 entries are null. - */ - NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "long"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt index 58c3352..90701ec 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalar.txt @@ -20,26 +20,116 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - /** - * Generated from template TimestampColumnCompareTimestampScalar.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type. The boolean output - * is stored in a separate boolean column. + * Generated from template TimestampColumnCompareTimestampScalar.txt, which covers binary comparison + * expressions between a column and a scalar. The boolean output is stored in a + * separate boolean column. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; - public (int colNum, Timestamp value, int outputColumn) { - super(colNum, new PisaTimestamp(value), outputColumn); + private int colNum; + private value; + private int outputColumn; + + public (int colNum, value, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #1 is type . + inputColVector1 = () batch.cols[colNum]; + + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; + + int[] sel = batch.selected; + boolean[] nullPos = inputColVector1.isNull; + boolean[] outNulls = outputColVector.isNull; + int n = batch.size; + long[] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + outputColVector.isRepeating = false; + outputColVector.noNulls = inputColVector1.noNulls; + if (inputColVector1.noNulls) { + if (inputColVector1.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + outputVector[0] = inputColVector1.compareTo(0, value) 0 ? 1 : 0; + outputColVector.isRepeating = true; + } else if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + outputVector[i] = inputColVector1.compareTo(i, value) 0 ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = inputColVector1.compareTo(i, value) 0 ? 1 : 0; + } + } + } else { + if (inputColVector1.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + if (!nullPos[0]) { + outputVector[0] = inputColVector1.compareTo(0, value) 0 ? 1 : 0; + outNulls[0] = false; + } else { + outNulls[0] = true; + } + outputColVector.isRepeating = true; + } else if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + outputVector[i] = inputColVector1.compareTo(i, value) 0 ? 1 : 0; + outNulls[i] = false; + } else { + //comparison with null is null + outNulls[i] = true; + } + } + } else { + System.arraycopy(nullPos, 0, outNulls, 0, n); + for(int i = 0; i != n; i++) { + if (!nullPos[i]) { + outputVector[i] = inputColVector1.compareTo(i, value) 0 ? 1 : 0; + } + } + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "long"; } @Override @@ -49,8 +139,8 @@ public class extends { VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("timestamp"), - VectorExpressionDescriptor.ArgumentType.getType("timestamp")) + VectorExpressionDescriptor.ArgumentType.getType(""), + VectorExpressionDescriptor.ArgumentType.getType("")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.COLUMN, VectorExpressionDescriptor.InputExpressionType.SCALAR).build(); diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalarBase.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalarBase.txt deleted file mode 100644 index ce940a4..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampColumnCompareTimestampScalarBase.txt +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - -/** - * Generated from template TimestampColumnCompareTimestampScalar.txt, which covers binary comparison - * expressions between a column and a scalar. The boolean output is stored in a - * separate boolean column. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - private int outputColumn; - - public (int colNum, PisaTimestamp value, int outputColumn) { - this.colNum = colNum; - this.value = value; - this.outputColumn = outputColumn; - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - TimestampColumnVector inputColVector1 = (TimestampColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; - int[] sel = batch.selected; - boolean[] nullPos = inputColVector1.isNull; - boolean[] outNulls = outputColVector.isNull; - int n = batch.size; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - outputColVector.isRepeating = false; - outputColVector.noNulls = inputColVector1.noNulls; - if (inputColVector1.noNulls) { - if (inputColVector1.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - outputVector[0] = inputColVector1.compareTo(0, value) 0 ? 1 : 0; - outputColVector.isRepeating = true; - } else if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputVector[i] = inputColVector1.compareTo(i, value) 0 ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = inputColVector1.compareTo(i, value) 0 ? 1 : 0; - } - } - } else { - if (inputColVector1.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - if (!nullPos[0]) { - outputVector[0] = inputColVector1.compareTo(0, value) 0 ? 1 : 0; - outNulls[0] = false; - } else { - outNulls[0] = true; - } - outputColVector.isRepeating = true; - } else if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - if (!nullPos[i]) { - outputVector[i] = inputColVector1.compareTo(i, value) 0 ? 1 : 0; - outNulls[i] = false; - } else { - //comparison with null is null - outNulls[i] = true; - } - } - } else { - System.arraycopy(nullPos, 0, outNulls, 0, n); - for(int i = 0; i != n; i++) { - if (!nullPos[i]) { - outputVector[i] = inputColVector1.compareTo(i, value) 0 ? 1 : 0; - } - } - } - } - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "long"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt index 8f89bd4..f958be8 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumn.txt @@ -19,9 +19,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -31,29 +30,129 @@ import org.apache.hadoop.hive.ql.exec.vector.*; * of these ColumnVector imports may be needed. Listing both of them * rather than using ....vectorization.*; */ -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.util.DateTimeMath; -import org.apache.hive.common.util.DateUtils; +import org.apache.hadoop.hive.serde2.io.DateWritable; /** - * Generated from template TimestampScalarArithmeticDateColumn.txt. + * Generated from template TimestampScalarArithmeticDateColumnBase.txt. * Implements a vectorized arithmetic operator with a scalar on the left and a * column vector on the right. The result is output to an output column vector. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; - public ( value, int colNum, int outputColumn) { - super(, colNum, outputColumn); + private int colNum; + private value; + private int outputColumn; + private Timestamp scratchTimestamp2; + private DateTimeMath dtm = new DateTimeMath(); + + public ( value, int colNum, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; + scratchTimestamp2 = new Timestamp(0); } public () { } @Override + /** + * Method to evaluate scalar-column operation in vectorized fashion. + * + * @batch a package of rows with each column stored in a vector + */ + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #2 is type date. + LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum]; + + // Output is type . + outputColVector = () batch.cols[outputColumn]; + + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector2.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector2.noNulls; + outputColVector.isRepeating = inputColVector2.isRepeating; + int n = batch.size; + + long[] vector2 = inputColVector2.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector2.isRepeating) { + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0])); + dtm.( + value, scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(0); + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + } else if (inputColVector2.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value, scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value, scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } + } else { /* there are nulls */ + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value, scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.( + value, scratchTimestamp2, outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + } + + NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return ""; + } + + @Override public VectorExpressionDescriptor.Descriptor getDescriptor() { return (new VectorExpressionDescriptor.Builder()) .setMode( @@ -61,7 +160,7 @@ public class extends { .setNumArguments(2) .setArgumentTypes( VectorExpressionDescriptor.ArgumentType.getType(""), - VectorExpressionDescriptor.ArgumentType.getType("")) + VectorExpressionDescriptor.ArgumentType.getType("date")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.SCALAR, VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumnBase.txt deleted file mode 100644 index 94be4f6..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticDateColumnBase.txt +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.*; - -import org.apache.hadoop.hive.common.type.PisaTimestamp; -/* - * Because of the templatized nature of the code, either or both - * of these ColumnVector imports may be needed. Listing both of them - * rather than using ....vectorization.*; - */ -import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.serde2.io.DateWritable; - -/** - * Generated from template TimestampScalarArithmeticDateColumnBase.txt. - * Implements a vectorized arithmetic operator with a scalar on the left and a - * column vector on the right. The result is output to an output column vector. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; - - public (PisaTimestamp value, int colNum, int outputColumn) { - this.colNum = colNum; - this.value = value; - this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); - } - - public () { - } - - @Override - /** - * Method to evaluate scalar-column operation in vectorized fashion. - * - * @batch a package of rows with each column stored in a vector - */ - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - // Input #2 is type date. - LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum]; - - // Output is type Timestamp. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; - - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector2.isNull; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector2.noNulls; - outputColVector.isRepeating = inputColVector2.isRepeating; - int n = batch.size; - - long[] vector2 = inputColVector2.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector2.isRepeating) { - outputColVector.( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])), - 0); - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector2.noNulls) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); - } - } - } else { /* there are nulls */ - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); - outputIsNull[i] = inputIsNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); - } - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - } - } - - NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "timestamp"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt index e9b9e67..585027a 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticIntervalYearMonthColumn.txt @@ -18,11 +18,13 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.common.type.PisaTimestamp; /* * Because of the templatized nature of the code, either or both * of these ColumnVector imports may be needed. Listing both of them @@ -44,16 +46,16 @@ public class extends VectorExpression { private static final long serialVersionUID = 1L; private int colNum; - private PisaTimestamp value; + private Timestamp value; private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; + private HiveIntervalYearMonth scratchIntervalYearMonth2; private DateTimeMath dtm = new DateTimeMath(); - public (PisaTimestamp value, int colNum, int outputColumn) { + public (Timestamp value, int colNum, int outputColumn) { this.colNum = colNum; this.value = value; this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); + scratchIntervalYearMonth2 = new HiveIntervalYearMonth(); } public () { @@ -72,61 +74,65 @@ public class extends VectorExpression { } // Input #2 is type Interval_Year_Month (months). - LongColumnVector inputColVector = (LongColumnVector) batch.cols[colNum]; + LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum]; // Output is type Timestamp. TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector.isNull; + boolean[] inputIsNull = inputColVector2.isNull; boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector.noNulls; - outputColVector.isRepeating = inputColVector.isRepeating; + outputColVector.noNulls = inputColVector2.noNulls; + outputColVector.isRepeating = inputColVector2.isRepeating; int n = batch.size; - long[] vector = inputColVector.vector; + long[] vector2 = inputColVector2.vector; // return immediately if batch is empty if (n == 0) { return; } - if (inputColVector.isRepeating) { - outputColVector.set(0, - dtm.addMonthsToPisaTimestamp(value, (int) vector[0], - scratchPisaTimestamp)); - + if (inputColVector2.isRepeating) { + scratchIntervalYearMonth2.set((int) vector2[0]); + dtm.( + value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector.noNulls) { + } else if (inputColVector2.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value, (int) vector[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value, (int) vector[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } } } else { /* there are nulls */ if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value, (int) vector[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, - dtm.addMonthsToPisaTimestamp(value, (int) vector[i], - scratchPisaTimestamp)); + scratchIntervalYearMonth2.set((int) vector2[i]); + dtm.( + value, scratchIntervalYearMonth2, outputColVector.getScratchTimestamp()); + outputColVector.setFromScratchTimestamp(i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt index 6725908..996c86a 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumn.txt @@ -19,10 +19,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hive.common.util.DateUtils; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -38,22 +36,113 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.util.DateTimeMath; /** - * Generated from template TimestampScalarArithmeticTimestampColumn.txt. + * Generated from template TimestampScalarArithmeticTimestampColumnBase.txt. * Implements a vectorized arithmetic operator with a scalar on the left and a * column vector on the right. The result is output to an output column vector. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; - public ( value, int colNum, int outputColumn) { - super(, colNum, outputColumn); + private int colNum; + private value; + private int outputColumn; + private DateTimeMath dtm = new DateTimeMath(); + + public ( value, int colNum, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; } public () { } @Override + /** + * Method to evaluate scalar-column operation in vectorized fashion. + * + * @batch a package of rows with each column stored in a vector + */ + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #2 is type . + inputColVector2 = () batch.cols[colNum]; + + // Output is type . + outputColVector = () batch.cols[outputColumn]; + + int[] sel = batch.selected; + boolean[] inputIsNull = inputColVector2.isNull; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = inputColVector2.noNulls; + outputColVector.isRepeating = inputColVector2.isRepeating; + int n = batch.size; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (inputColVector2.isRepeating) { + dtm.( + value, inputColVector2.asScratch(0), outputColVector.getScratch()); + outputColVector.setFromScratch(0); + // Even if there are no nulls, we always copy over entry 0. Simplifies code. + outputIsNull[0] = inputIsNull[0]; + } else if (inputColVector2.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + value, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + value, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + + } + } + } else { /* there are nulls */ + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + dtm.( + value, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + outputIsNull[i] = inputIsNull[i]; + } + } else { + for(int i = 0; i != n; i++) { + dtm.( + value, inputColVector2.asScratch(i), outputColVector.getScratch()); + outputColVector.setFromScratch(i); + } + System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); + } + } + + NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "timestamp"; + } + + @Override public VectorExpressionDescriptor.Descriptor getDescriptor() { return (new VectorExpressionDescriptor.Builder()) .setMode( diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumnBase.txt deleted file mode 100644 index 0ff9226..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarArithmeticTimestampColumnBase.txt +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; -import org.apache.hadoop.hive.ql.exec.vector.*; - -import org.apache.hadoop.hive.common.type.PisaTimestamp; -/* - * Because of the templatized nature of the code, either or both - * of these ColumnVector imports may be needed. Listing both of them - * rather than using ....vectorization.*; - */ -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; -import org.apache.hadoop.hive.ql.util.DateTimeMath; - -/** - * Generated from template TimestampScalarArithmeticTimestampColumnBase.txt. - * Implements a vectorized arithmetic operator with a scalar on the left and a - * column vector on the right. The result is output to an output column vector. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; - private DateTimeMath dtm = new DateTimeMath(); - - public (PisaTimestamp value, int colNum, int outputColumn) { - this.colNum = colNum; - this.value = value; - this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); - } - - public () { - } - - @Override - /** - * Method to evaluate scalar-column operation in vectorized fashion. - * - * @batch a package of rows with each column stored in a vector - */ - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - // Input #2 is type timestamp/interval_day_time. - TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum]; - - // Output is type timestamp/interval_day_time. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; - - int[] sel = batch.selected; - boolean[] inputIsNull = inputColVector2.isNull; - boolean[] outputIsNull = outputColVector.isNull; - outputColVector.noNulls = inputColVector2.noNulls; - outputColVector.isRepeating = inputColVector2.isRepeating; - int n = batch.size; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - if (inputColVector2.isRepeating) { - outputColVector.( - value, inputColVector2.asScratchPisaTimestamp(0), 0); - - // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; - } else if (inputColVector2.noNulls) { - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - value, inputColVector2.asScratchPisaTimestamp(i), i); - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - value, inputColVector2.asScratchPisaTimestamp(i), i); - } - } - } else { /* there are nulls */ - if (batch.selectedInUse) { - for(int j = 0; j != n; j++) { - int i = sel[j]; - outputColVector.( - value, inputColVector2.asScratchPisaTimestamp(i), i); - outputIsNull[i] = inputIsNull[i]; - } - } else { - for(int i = 0; i != n; i++) { - outputColVector.( - value, inputColVector2.asScratchPisaTimestamp(i), i); - } - System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); - } - } - - NullUtil.setNullOutputEntriesColScalar(outputColVector, batch.selectedInUse, sel, n); - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "timestamp"; - } -} diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt index 9e855e8..6815b5b 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareLongDoubleColumn.txt @@ -19,8 +19,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.expressions.*; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; @@ -35,7 +35,7 @@ public class extends { private static final long serialVersionUID = 1L; public (Timestamp value, int colNum, int outputColumn) { - super(new PisaTimestamp(value).(), colNum, outputColumn); + super(TimestampColumnVector.(value), colNum, outputColumn); } public () { diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt index df9f3c9..6506c93 100644 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt +++ ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumn.txt @@ -21,26 +21,117 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; import java.sql.Timestamp; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - /** - * Generated from template TimestampColumnCompareTimestampScalar.txt, which covers comparison - * expressions between a datetime/interval column and a scalar of the same type. The boolean output - * is stored in a separate boolean column. + * Generated from template ScalarCompareTimestamp.txt, which covers comparison + * expressions between a long/double scalar and a column. The boolean output is stored in a + * separate boolean column. */ -public class extends { +public class extends VectorExpression { private static final long serialVersionUID = 1L; - public (Timestamp value, int colNum, int outputColumn) { - super(new PisaTimestamp(value), colNum, outputColumn); + private int colNum; + private value; + private int outputColumn; + + public ( value, int colNum, int outputColumn) { + this.colNum = colNum; + this.value = value; + this.outputColumn = outputColumn; } public () { - super(); + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + // Input #2 is type . + inputColVector2 = () batch.cols[colNum]; + + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; + + int[] sel = batch.selected; + boolean[] nullPos = inputColVector2.isNull; + boolean[] outNulls = outputColVector.isNull; + int n = batch.size; + long[] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + outputColVector.isRepeating = false; + outputColVector.noNulls = inputColVector2.noNulls; + if (inputColVector2.noNulls) { + if (inputColVector2.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + outputVector[0] = inputColVector2.compareTo(value, 0) 0 ? 1 : 0; + outputColVector.isRepeating = true; + } else if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + outputVector[i] = inputColVector2.compareTo(value, i) 0 ? 1 : 0; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = inputColVector2.compareTo(value, i) 0 ? 1 : 0; + } + } + } else { + if (inputColVector2.isRepeating) { + //All must be selected otherwise size would be zero + //Repeating property will not change. + if (!nullPos[0]) { + outputVector[0] = inputColVector2.compareTo(value, 0) 0 ? 1 : 0; + outNulls[0] = false; + } else { + outNulls[0] = true; + } + outputColVector.isRepeating = true; + } else if (batch.selectedInUse) { + for(int j=0; j != n; j++) { + int i = sel[j]; + if (!nullPos[i]) { + outputVector[i] = inputColVector2.compareTo(value, i) 0 ? 1 : 0; + outNulls[i] = false; + } else { + //comparison with null is null + outNulls[i] = true; + } + } + } else { + System.arraycopy(nullPos, 0, outNulls, 0, n); + for(int i = 0; i != n; i++) { + if (!nullPos[i]) { + outputVector[i] = inputColVector2.compareTo(value, i) 0 ? 1 : 0; + } + } + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "long"; } @Override @@ -50,8 +141,8 @@ public class extends { VectorExpressionDescriptor.Mode.PROJECTION) .setNumArguments(2) .setArgumentTypes( - VectorExpressionDescriptor.ArgumentType.getType("timestamp"), - VectorExpressionDescriptor.ArgumentType.getType("timestamp")) + VectorExpressionDescriptor.ArgumentType.getType(""), + VectorExpressionDescriptor.ArgumentType.getType("")) .setInputExpressionTypes( VectorExpressionDescriptor.InputExpressionType.SCALAR, VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); diff --git ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumnBase.txt ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumnBase.txt deleted file mode 100644 index bd345e7..0000000 --- ql/src/gen/vectorization/ExpressionTemplates/TimestampScalarCompareTimestampColumnBase.txt +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector.expressions.gen; - -import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; - -import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; - -/** - * Generated from template ScalarCompareTimestamp.txt, which covers comparison - * expressions between a long/double scalar and a column. The boolean output is stored in a - * separate boolean column. - */ -public abstract class extends VectorExpression { - - private static final long serialVersionUID = 1L; - - private int colNum; - private PisaTimestamp value; - private int outputColumn; - - public (PisaTimestamp value, int colNum, int outputColumn) { - this.colNum = colNum; - this.value = value; - this.outputColumn = outputColumn; - } - - public () { - } - - @Override - public void evaluate(VectorizedRowBatch batch) { - - if (childExpressions != null) { - super.evaluateChildren(batch); - } - - TimestampColumnVector inputColVector2 = (TimestampColumnVector) batch.cols[colNum]; - LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumn]; - int[] sel = batch.selected; - boolean[] nullPos = inputColVector2.isNull; - boolean[] outNulls = outputColVector.isNull; - int n = batch.size; - long[] outputVector = outputColVector.vector; - - // return immediately if batch is empty - if (n == 0) { - return; - } - - outputColVector.isRepeating = false; - outputColVector.noNulls = inputColVector2.noNulls; - if (inputColVector2.noNulls) { - if (inputColVector2.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - outputVector[0] = inputColVector2.compareTo(value, 0) 0 ? 1 : 0; - outputColVector.isRepeating = true; - } else if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - outputVector[i] = inputColVector2.compareTo(value, i) 0 ? 1 : 0; - } - } else { - for(int i = 0; i != n; i++) { - outputVector[i] = inputColVector2.compareTo(value, i) 0 ? 1 : 0; - } - } - } else { - if (inputColVector2.isRepeating) { - //All must be selected otherwise size would be zero - //Repeating property will not change. - if (!nullPos[0]) { - outputVector[0] = inputColVector2.compareTo(value, 0) 0 ? 1 : 0; - outNulls[0] = false; - } else { - outNulls[0] = true; - } - outputColVector.isRepeating = true; - } else if (batch.selectedInUse) { - for(int j=0; j != n; j++) { - int i = sel[j]; - if (!nullPos[i]) { - outputVector[i] = inputColVector2.compareTo(value, i) 0 ? 1 : 0; - outNulls[i] = false; - } else { - //comparison with null is null - outNulls[i] = true; - } - } - } else { - System.arraycopy(nullPos, 0, outNulls, 0, n); - for(int i = 0; i != n; i++) { - if (!nullPos[i]) { - outputVector[i] = inputColVector2.compareTo(value, i) 0 ? 1 : 0; - } - } - } - } - } - - @Override - public int getOutputColumn() { - return outputColumn; - } - - @Override - public String getOutputType() { - return "long"; - } -} diff --git ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt index 3cdf405..7e34965 100644 --- ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt +++ ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt @@ -18,7 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; @@ -49,7 +50,7 @@ public class extends VectorAggregateExpression { private static final long serialVersionUID = 1L; - transient private final PisaTimestamp value; + transient private final Timestamp value; /** * Value is explicitly (re)initialized in reset() @@ -57,15 +58,15 @@ public class extends VectorAggregateExpression { transient private boolean isNull = true; public Aggregation() { - value = new PisaTimestamp(); + value = new Timestamp(0); } public void checkValue(TimestampColumnVector colVector, int index) { if (isNull) { isNull = false; - colVector.pisaTimestampUpdate(this.value, index); + colVector.timestampUpdate(this.value, index); } else if (colVector.compareTo(this.value, index) 0) { - colVector.pisaTimestampUpdate(this.value, index); + colVector.timestampUpdate(this.value, index); } } @@ -77,7 +78,7 @@ public class extends VectorAggregateExpression { @Override public void reset () { isNull = true; - this.value.reset(); + this.value.setTime(0); } } @@ -311,7 +312,7 @@ public class extends VectorAggregateExpression { if (inputColVector.noNulls && (myagg.isNull || (inputColVector.compareTo(myagg.value, 0) 0))) { myagg.isNull = false; - inputColVector.pisaTimestampUpdate(myagg.value, 0); + inputColVector.timestampUpdate(myagg.value, 0); } return; } @@ -344,10 +345,10 @@ public class extends VectorAggregateExpression { if (!isNull[i]) { if (myagg.isNull) { myagg.isNull = false; - inputColVector.pisaTimestampUpdate(myagg.value, i); + inputColVector.timestampUpdate(myagg.value, i); } else if (inputColVector.compareTo(myagg.value, i) 0) { - inputColVector.pisaTimestampUpdate(myagg.value, i); + inputColVector.timestampUpdate(myagg.value, i); } } } @@ -360,14 +361,14 @@ public class extends VectorAggregateExpression { int[] selected) { if (myagg.isNull) { - inputColVector.pisaTimestampUpdate(myagg.value, selected[0]); + inputColVector.timestampUpdate(myagg.value, selected[0]); myagg.isNull = false; } for (int i=0; i< batchSize; ++i) { int sel = selected[i]; if (inputColVector.compareTo(myagg.value, sel) 0) { - inputColVector.pisaTimestampUpdate(myagg.value, sel); + inputColVector.timestampUpdate(myagg.value, sel); } } } @@ -381,11 +382,11 @@ public class extends VectorAggregateExpression { for(int i=0;i 0) { - inputColVector.pisaTimestampUpdate(myagg.value, i); + inputColVector.timestampUpdate(myagg.value, i); } } } @@ -396,13 +397,13 @@ public class extends VectorAggregateExpression { TimestampColumnVector inputColVector, int batchSize) { if (myagg.isNull) { - inputColVector.pisaTimestampUpdate(myagg.value, 0); + inputColVector.timestampUpdate(myagg.value, 0); myagg.isNull = false; } for (int i=0;i 0) { - inputColVector.pisaTimestampUpdate(myagg.value, i); + inputColVector.timestampUpdate(myagg.value, i); } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java index 5de055c..bb795fa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java @@ -21,6 +21,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; public final class TimestampUtils { @@ -38,4 +39,11 @@ public static TimestampWritable timestampColumnVectorWritable( timestampWritable.set(timestampColVector.asScratchTimestamp(elementNum)); return timestampWritable; } + + public static HiveIntervalDayTimeWritable intervalDayTimeColumnVectorWritable( + IntervalDayTimeColumnVector intervalDayTimeColVector, int elementNum, + HiveIntervalDayTimeWritable intervalDayTimeWritable) { + intervalDayTimeWritable.set(intervalDayTimeColVector.asScratchIntervalDayTime(elementNum)); + return intervalDayTimeWritable; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java index 965c027..de0300a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java @@ -288,7 +288,26 @@ void assign(int batchIndex, Object object) { } } - private class IntervalDayTimeAssigner extends AbstractTimestampAssigner { + private abstract class AbstractIntervalDayTimeAssigner extends Assigner { + + protected IntervalDayTimeColumnVector colVector; + + AbstractIntervalDayTimeAssigner(int columnIndex) { + super(columnIndex); + } + + @Override + void setColumnVector(VectorizedRowBatch batch) { + colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex]; + } + + @Override + void forgetColumnVector() { + colVector = null; + } + } + + private class IntervalDayTimeAssigner extends AbstractIntervalDayTimeAssigner { IntervalDayTimeAssigner(int columnIndex) { super(columnIndex); @@ -301,7 +320,7 @@ void assign(int batchIndex, Object object) { } else { HiveIntervalDayTimeWritable idtw = (HiveIntervalDayTimeWritable) object; HiveIntervalDayTime idt = idtw.getHiveIntervalDayTime(); - colVector.set(batchIndex, idt.pisaTimestampUpdate(colVector.useScratchPisaTimestamp())); + colVector.set(batchIndex, idt); colVector.isNull[batchIndex] = false; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java index 463c8a6..96b8f78 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.io.ByteWritable; @@ -176,6 +177,16 @@ protected void assignTimestamp(TimestampWritable tw, int index) { } } + private static abstract class VectorIntervalDayTimeColumnAssign + extends VectorColumnAssignVectorBase { + + protected void assignIntervalDayTime(HiveIntervalDayTime value, int index) { + outCol.set(index, value); + } + protected void assignIntervalDayTime(HiveIntervalDayTimeWritable tw, int index) { + outCol.set(index, tw.getHiveIntervalDayTime()); + } + } public static VectorColumnAssign[] buildAssigners(VectorizedRowBatch outputBatch) throws HiveException { @@ -364,7 +375,7 @@ public void assignObjectValue(Object val, int destIndex) throws HiveException { } }.init(outputBatch, (LongColumnVector) destCol); break; - case INTERVAL_DAY_TIME:outVCA = new VectorLongColumnAssign() { + case INTERVAL_DAY_TIME:outVCA = new VectorIntervalDayTimeColumnAssign() { @Override public void assignObjectValue(Object val, int destIndex) throws HiveException { if (val == null) { @@ -372,12 +383,12 @@ public void assignObjectValue(Object val, int destIndex) throws HiveException { } else { HiveIntervalDayTimeWritable bw = (HiveIntervalDayTimeWritable) val; - assignLong( - DateUtils.getIntervalDayTimeTotalNanos(bw.getHiveIntervalDayTime()), + assignIntervalDayTime( + bw.getHiveIntervalDayTime(), destIndex); } } - }.init(outputBatch, (LongColumnVector) destCol); + }.init(outputBatch, (IntervalDayTimeColumnVector) destCol); break; default: throw new HiveException("Incompatible Long vector column and primitive category " + diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java index 0949145..935b47b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java @@ -60,6 +60,11 @@ protected int[] timestampIndices; /** + * indices of INTERVAL_DAY_TIME primitive keys. + */ + protected int[] intervalDayTimeIndices; + + /** * Helper class for looking up a key value based on key index. */ public class KeyLookupHelper { @@ -68,12 +73,13 @@ public int stringIndex; public int decimalIndex; public int timestampIndex; + public int intervalDayTimeIndex; private static final int INDEX_UNUSED = -1; private void resetIndices() { this.longIndex = this.doubleIndex = this.stringIndex = this.decimalIndex = - timestampIndex = INDEX_UNUSED; + timestampIndex = intervalDayTimeIndex = INDEX_UNUSED; } public void setLong(int index) { resetIndices(); @@ -99,6 +105,11 @@ public void setTimestamp(int index) { resetIndices(); this.timestampIndex= index; } + + public void setIntervalDayTime(int index) { + resetIndices(); + this.intervalDayTimeIndex= index; + } } /** @@ -114,6 +125,7 @@ public void setTimestamp(int index) { protected int stringIndicesIndex; protected int decimalIndicesIndex; protected int timestampIndicesIndex; + protected int intervalDayTimeIndicesIndex; protected VectorColumnSetInfo(int keyCount) { this.keyCount = keyCount; @@ -130,6 +142,8 @@ protected VectorColumnSetInfo(int keyCount) { decimalIndicesIndex = 0; timestampIndices = new int[this.keyCount]; timestampIndicesIndex = 0; + intervalDayTimeIndices = new int[this.keyCount]; + intervalDayTimeIndicesIndex = 0; indexLookup = new KeyLookupHelper[this.keyCount]; } @@ -172,6 +186,12 @@ protected void addKey(String outputType) throws HiveException { ++timestampIndicesIndex; break; + case INTERVAL_DAY_TIME: + intervalDayTimeIndices[intervalDayTimeIndicesIndex] = addIndex; + indexLookup[addIndex].setIntervalDayTime(intervalDayTimeIndicesIndex); + ++intervalDayTimeIndicesIndex; + break; + default: throw new HiveException("Unexpected column vector type " + columnVectorType); } @@ -185,5 +205,6 @@ protected void finishAdding() { stringIndices = Arrays.copyOf(stringIndices, stringIndicesIndex); decimalIndices = Arrays.copyOf(decimalIndices, decimalIndicesIndex); timestampIndices = Arrays.copyOf(timestampIndices, timestampIndicesIndex); + intervalDayTimeIndices = Arrays.copyOf(intervalDayTimeIndices, intervalDayTimeIndicesIndex); } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java index 73476a3..c8e0284 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java @@ -223,6 +223,34 @@ void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBa } } } + + private class IntervalDayTimeCopyRow extends CopyRow { + + IntervalDayTimeCopyRow(int inColumnIndex, int outColumnIndex) { + super(inColumnIndex, outColumnIndex); + } + + @Override + void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBatch, int outBatchIndex) { + IntervalDayTimeColumnVector inColVector = (IntervalDayTimeColumnVector) inBatch.cols[inColumnIndex]; + IntervalDayTimeColumnVector outColVector = (IntervalDayTimeColumnVector) outBatch.cols[outColumnIndex]; + + if (inColVector.isRepeating) { + if (inColVector.noNulls || !inColVector.isNull[0]) { + outColVector.setElement(outBatchIndex, 0, inColVector); + } else { + VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); + } + } else { + if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) { + outColVector.setElement(outBatchIndex, inBatchIndex, inColVector); + } else { + VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); + } + } + } + } + private CopyRow[] subRowToBatchCopiersByValue; private CopyRow[] subRowToBatchCopiersByReference; @@ -250,6 +278,10 @@ public void init(VectorColumnMapping columnMapping) throws HiveException { copyRowByValue = new TimestampCopyRow(inputColumn, outputColumn); break; + case INTERVAL_DAY_TIME: + copyRowByValue = new IntervalDayTimeCopyRow(inputColumn, outputColumn); + break; + case DOUBLE: copyRowByValue = new DoubleCopyRow(inputColumn, outputColumn); break; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java index 50881e7..3eadc12 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java @@ -264,7 +264,14 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } } - private class IntervalDayTimeReader extends AbstractTimestampReader { + private abstract class AbstractIntervalDayTimeReader extends Reader { + + AbstractIntervalDayTimeReader(int columnIndex) { + super(columnIndex); + } + } + + private class IntervalDayTimeReader extends AbstractIntervalDayTimeReader { DeserializeRead.ReadIntervalDayTimeResults readIntervalDayTimeResults; @@ -275,14 +282,14 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { @Override void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { - TimestampColumnVector colVector = (TimestampColumnVector) batch.cols[columnIndex]; + IntervalDayTimeColumnVector colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex]; if (deserializeRead.readCheckNull()) { VectorizedBatchUtil.setNullColIsNullValue(colVector, batchIndex); } else { deserializeRead.readIntervalDayTime(readIntervalDayTimeResults); HiveIntervalDayTime idt = readIntervalDayTimeResults.getHiveIntervalDayTime(); - colVector.set(batchIndex, idt.pisaTimestampUpdate(colVector.useScratchPisaTimestamp())); + colVector.set(batchIndex, idt); colVector.isNull[batchIndex] = false; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java index 0b9ad55..7b3f781 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java @@ -59,6 +59,9 @@ // TimestampColumnVector --> // TIMESTAMP // + // IntervalDayTimeColumnVector --> + // INTERVAL_DAY_TIME + // public enum ArgumentType { NONE (0x000), INT_FAMILY (0x001), @@ -76,7 +79,6 @@ INTERVAL_FAMILY (INTERVAL_YEAR_MONTH.value | INTERVAL_DAY_TIME.value), INT_INTERVAL_YEAR_MONTH (INT_FAMILY.value | INTERVAL_YEAR_MONTH.value), INT_DATE_INTERVAL_YEAR_MONTH (INT_FAMILY.value | DATE.value | INTERVAL_YEAR_MONTH.value), - TIMESTAMP_INTERVAL_DAY_TIME (TIMESTAMP.value | INTERVAL_DAY_TIME.value), STRING_DATETIME_FAMILY (STRING_FAMILY.value | DATETIME_FAMILY.value), ALL_FAMILY (0xFFF); @@ -346,7 +348,7 @@ public String toString() { return ve; } } catch (Exception ex) { - throw new HiveException(ex); + throw new HiveException("Could not instantiate VectorExpression class " + ve.getSimpleName(), ex); } } if (LOG.isDebugEnabled()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java index 622f4a3..e883f38 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -323,7 +322,26 @@ Object extract(int batchIndex) { } } - private class IntervalDayTimeExtractor extends AbstractTimestampExtractor { + private abstract class AbstractIntervalDayTimeExtractor extends Extractor { + + protected IntervalDayTimeColumnVector colVector; + + AbstractIntervalDayTimeExtractor(int columnIndex) { + super(columnIndex); + } + + @Override + void setColumnVector(VectorizedRowBatch batch) { + colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex]; + } + + @Override + void forgetColumnVector() { + colVector = null; + } + } + + private class IntervalDayTimeExtractor extends AbstractIntervalDayTimeExtractor { private HiveIntervalDayTime hiveIntervalDayTime; @@ -337,7 +355,7 @@ Object extract(int batchIndex) { Object extract(int batchIndex) { int adjustedIndex = (colVector.isRepeating ? 0 : batchIndex); if (colVector.noNulls || !colVector.isNull[adjustedIndex]) { - hiveIntervalDayTime.set(colVector.asScratchPisaTimestamp(adjustedIndex)); + hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(adjustedIndex)); PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector.set(object, hiveIntervalDayTime); return object; } else { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java index 9f0ac11..50d0452 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java @@ -131,5 +131,17 @@ public void copyGroupKey(VectorizedRowBatch inputBatch, VectorizedRowBatch outpu outputColumnVector.isNull[outputBatch.size] = true; } } + for(int i=0;i 0 ? new long[longValuesCount] : EMPTY_LONG_ARRAY; doubleValues = doubleValuesCount > 0 ? new double[doubleValuesCount] : EMPTY_DOUBLE_ARRAY; decimalValues = decimalValuesCount > 0 ? new HiveDecimalWritable[decimalValuesCount] : EMPTY_DECIMAL_ARRAY; - timestampValues = timestampValuesCount > 0 ? new PisaTimestamp[timestampValuesCount] : EMPTY_TIMESTAMP_ARRAY; + timestampValues = timestampValuesCount > 0 ? new Timestamp[timestampValuesCount] : EMPTY_TIMESTAMP_ARRAY; + intervalDayTimeValues = intervalDayTimeValuesCount > 0 ? new HiveIntervalDayTime[intervalDayTimeValuesCount] : EMPTY_INTERVAL_DAY_TIME_ARRAY; for(int i = 0; i < decimalValuesCount; ++i) { decimalValues[i] = new HiveDecimalWritable(HiveDecimal.ZERO); } @@ -79,10 +84,13 @@ public VectorHashKeyWrapper(int longValuesCount, int doubleValuesCount, byteLengths = EMPTY_INT_ARRAY; } for(int i = 0; i < timestampValuesCount; ++i) { - timestampValues[i] = new PisaTimestamp(); + timestampValues[i] = new Timestamp(0); + } + for(int i = 0; i < intervalDayTimeValuesCount; ++i) { + intervalDayTimeValues[i] = new HiveIntervalDayTime(); } isNull = new boolean[longValuesCount + doubleValuesCount + byteValuesCount + - decimalValuesCount + timestampValuesCount]; + decimalValuesCount + timestampValuesCount + intervalDayTimeValuesCount]; hashcode = 0; } @@ -108,6 +116,10 @@ public void setHashKey() { hashcode ^= timestampValues[i].hashCode(); } + for (int i = 0; i < intervalDayTimeValues.length; i++) { + hashcode ^= intervalDayTimeValues[i].hashCode(); + } + // This code, with branches and all, is not executed if there are no string keys for (int i = 0; i < byteValues.length; ++i) { /* @@ -146,6 +158,7 @@ public boolean equals(Object that) { Arrays.equals(doubleValues, keyThat.doubleValues) && Arrays.equals(decimalValues, keyThat.decimalValues) && Arrays.equals(timestampValues, keyThat.timestampValues) && + Arrays.equals(intervalDayTimeValues, keyThat.intervalDayTimeValues) && Arrays.equals(isNull, keyThat.isNull) && byteValues.length == keyThat.byteValues.length && (0 == byteValues.length || bytesEquals(keyThat)); @@ -212,14 +225,21 @@ public void duplicateTo(VectorHashKeyWrapper clone) { clone.byteLengths = EMPTY_INT_ARRAY; } if (timestampValues.length > 0) { - clone.timestampValues = new PisaTimestamp[timestampValues.length]; + clone.timestampValues = new Timestamp[timestampValues.length]; for(int i = 0; i < timestampValues.length; ++i) { - clone.timestampValues[i] = new PisaTimestamp(); - clone.timestampValues[i].update(timestampValues[i]); + clone.timestampValues[i] = (Timestamp) timestampValues[i].clone(); } } else { clone.timestampValues = EMPTY_TIMESTAMP_ARRAY; } + if (intervalDayTimeValues.length > 0) { + clone.intervalDayTimeValues = new HiveIntervalDayTime[intervalDayTimeValues.length]; + for(int i = 0; i < intervalDayTimeValues.length; ++i) { + clone.intervalDayTimeValues[i] = (HiveIntervalDayTime) intervalDayTimeValues[i].clone(); + } + } else { + clone.intervalDayTimeValues = EMPTY_INTERVAL_DAY_TIME_ARRAY; + } clone.hashcode = hashcode; assert clone.equals(this); @@ -281,14 +301,14 @@ public void assignNullDecimal(int index) { isNull[longValues.length + doubleValues.length + byteValues.length + index] = true; } - public void assignTimestamp(int index, PisaTimestamp value) { - timestampValues[index].update(value); + public void assignTimestamp(int index, Timestamp value) { + timestampValues[index] = value; isNull[longValues.length + doubleValues.length + byteValues.length + decimalValues.length + index] = false; } public void assignTimestamp(int index, TimestampColumnVector colVector, int elementNum) { - colVector.pisaTimestampUpdate(timestampValues[index], elementNum); + colVector.timestampUpdate(timestampValues[index], elementNum); isNull[longValues.length + doubleValues.length + byteValues.length + decimalValues.length + index] = false; } @@ -298,15 +318,33 @@ public void assignNullTimestamp(int index) { decimalValues.length + index] = true; } + public void assignIntervalDayTime(int index, HiveIntervalDayTime value) { + intervalDayTimeValues[index].set(value); + isNull[longValues.length + doubleValues.length + byteValues.length + + decimalValues.length + timestampValues.length + index] = false; + } + + public void assignIntervalDayTime(int index, IntervalDayTimeColumnVector colVector, int elementNum) { + intervalDayTimeValues[index].set(colVector.asScratchIntervalDayTime(elementNum)); + isNull[longValues.length + doubleValues.length + byteValues.length + + decimalValues.length + timestampValues.length + index] = false; + } + + public void assignNullIntervalDayTime(int index) { + isNull[longValues.length + doubleValues.length + byteValues.length + + decimalValues.length + timestampValues.length + index] = true; + } + @Override public String toString() { - return String.format("%d[%s] %d[%s] %d[%s] %d[%s] %d[%s]", + return String.format("%d[%s] %d[%s] %d[%s] %d[%s] %d[%s] %d[%s]", longValues.length, Arrays.toString(longValues), doubleValues.length, Arrays.toString(doubleValues), byteValues.length, Arrays.toString(byteValues), decimalValues.length, Arrays.toString(decimalValues), - timestampValues.length, Arrays.toString(timestampValues)); + timestampValues.length, Arrays.toString(timestampValues), + intervalDayTimeValues.length, Arrays.toString(intervalDayTimeValues)); } public boolean getIsLongNull(int i) { @@ -364,9 +402,17 @@ public boolean getIsTimestampNull(int i) { decimalValues.length + i]; } - public PisaTimestamp getTimestamp(int i) { + public Timestamp getTimestamp(int i) { return timestampValues[i]; } + public boolean getIsIntervalDayTimeNull(int i) { + return isNull[longValues.length + doubleValues.length + byteValues.length + + decimalValues.length + timestampValues.length + i]; + } + + public HiveIntervalDayTime getIntervalDayTime(int i) { + return intervalDayTimeValues[i]; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java index 1c34124..bfd26ae 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java @@ -198,6 +198,28 @@ public void evaluateBatch(VectorizedRowBatch batch) throws HiveException { columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse)); } } + for(int i=0;i= 0) { + return kw.getIsIntervalDayTimeNull(klh.intervalDayTimeIndex)? null : + keyOutputWriter.writeValue( + kw.getIntervalDayTime(klh.intervalDayTimeIndex)); + } else { throw new HiveException(String.format( - "Internal inconsistent KeyLookupHelper at index [%d]:%d %d %d %d %d", + "Internal inconsistent KeyLookupHelper at index [%d]:%d %d %d %d %d %d", i, klh.longIndex, klh.doubleIndex, klh.stringIndex, klh.decimalIndex, - klh.timestampIndex)); + klh.timestampIndex, klh.intervalDayTimeIndex)); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java index dea38e8..6af3d99 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java @@ -23,7 +23,6 @@ import java.util.List; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; @@ -332,11 +331,11 @@ boolean apply(VectorizedRowBatch batch, int batchIndex) throws IOException { @Override boolean apply(VectorizedRowBatch batch, int batchIndex) throws IOException { - TimestampColumnVector colVector = (TimestampColumnVector) batch.cols[columnIndex]; + IntervalDayTimeColumnVector colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex]; if (colVector.isRepeating) { if (colVector.noNulls || !colVector.isNull[0]) { - hiveIntervalDayTime.set(colVector.asScratchPisaTimestamp(0)); + hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(0)); serializeWrite.writeHiveIntervalDayTime(hiveIntervalDayTime); return true; } else { @@ -345,7 +344,7 @@ boolean apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } } else { if (colVector.noNulls || !colVector.isNull[batchIndex]) { - hiveIntervalDayTime.set(colVector.asScratchPisaTimestamp(batchIndex)); + hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(batchIndex)); serializeWrite.writeHiveIntervalDayTime(hiveIntervalDayTime); return true; } else { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index dd59bf2..d72da55 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -2330,10 +2330,12 @@ public static String mapTypeNameSynonyms(String typeName) { case INTERVAL_YEAR_MONTH: return ColumnVector.Type.LONG; - case INTERVAL_DAY_TIME: case TIMESTAMP: return ColumnVector.Type.TIMESTAMP; + case INTERVAL_DAY_TIME: + return ColumnVector.Type.INTERVAL_DAY_TIME; + case FLOAT: case DOUBLE: return ColumnVector.Type.DOUBLE; @@ -2366,19 +2368,19 @@ public static String mapTypeNameSynonyms(String typeName) { add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null, VectorUDAFMinDouble.class)); add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null, VectorUDAFMinString.class)); add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFMinDecimal.class)); - add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.TIMESTAMP_INTERVAL_DAY_TIME, null, VectorUDAFMinTimestamp.class)); + add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, null, VectorUDAFMinTimestamp.class)); add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.INT_DATE_INTERVAL_YEAR_MONTH, null, VectorUDAFMaxLong.class)); add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null, VectorUDAFMaxDouble.class)); add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null, VectorUDAFMaxString.class)); add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFMaxDecimal.class)); - add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.TIMESTAMP_INTERVAL_DAY_TIME, null, VectorUDAFMaxTimestamp.class)); + add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, null, VectorUDAFMaxTimestamp.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.NONE, GroupByDesc.Mode.HASH, VectorUDAFCountStar.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INT_DATE_INTERVAL_YEAR_MONTH, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.MERGEPARTIAL, VectorUDAFCountMerge.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); - add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.TIMESTAMP_INTERVAL_DAY_TIME, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); + add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); add(new AggregateDefinition("sum", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, null, VectorUDAFSumLong.class)); add(new AggregateDefinition("sum", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null, VectorUDAFSumDouble.class)); add(new AggregateDefinition("sum", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFSumDecimal.class)); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java index a68d0cc..be04da8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java @@ -144,9 +144,10 @@ public static ColumnVector createColumnVector(TypeInfo typeInfo) { case DATE: case INTERVAL_YEAR_MONTH: return new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE); - case INTERVAL_DAY_TIME: case TIMESTAMP: return new TimestampColumnVector(VectorizedRowBatch.DEFAULT_SIZE); + case INTERVAL_DAY_TIME: + return new IntervalDayTimeColumnVector(VectorizedRowBatch.DEFAULT_SIZE); case FLOAT: case DOUBLE: return new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE); @@ -417,14 +418,14 @@ private static void setVector(Object row, } break; case INTERVAL_DAY_TIME: { - LongColumnVector lcv = (LongColumnVector) batch.cols[offset + colIndex]; + IntervalDayTimeColumnVector icv = (IntervalDayTimeColumnVector) batch.cols[offset + colIndex]; if (writableCol != null) { - HiveIntervalDayTime i = ((HiveIntervalDayTimeWritable) writableCol).getHiveIntervalDayTime(); - lcv.vector[rowIndex] = DateUtils.getIntervalDayTimeTotalNanos(i); - lcv.isNull[rowIndex] = false; + HiveIntervalDayTime idt = ((HiveIntervalDayTimeWritable) writableCol).getHiveIntervalDayTime(); + icv.set(rowIndex, idt); + icv.isNull[rowIndex] = false; } else { - lcv.vector[rowIndex] = 1; - setNullColIsNullValue(lcv, rowIndex); + icv.setNullValue(rowIndex); + setNullColIsNullValue(icv, rowIndex); } } break; @@ -585,6 +586,8 @@ static ColumnVector cloneColumnVector(ColumnVector source decColVector.scale); } else if (source instanceof TimestampColumnVector) { return new TimestampColumnVector(((TimestampColumnVector) source).getLength()); + } else if (source instanceof IntervalDayTimeColumnVector) { + return new IntervalDayTimeColumnVector(((IntervalDayTimeColumnVector) source).getLength()); } else if (source instanceof ListColumnVector) { ListColumnVector src = (ListColumnVector) source; ColumnVector child = cloneColumnVector(src.child); @@ -688,6 +691,9 @@ public static void debugDisplayOneRow(VectorizedRowBatch batch, int index, Strin Timestamp timestamp = new Timestamp(0); ((TimestampColumnVector) colVector).timestampUpdate(timestamp, index); sb.append(timestamp.toString()); + } else if (colVector instanceof IntervalDayTimeColumnVector) { + HiveIntervalDayTime intervalDayTime = ((IntervalDayTimeColumnVector) colVector).asScratchIntervalDayTime(index); + sb.append(intervalDayTime.toString()); } else { sb.append("Unknown"); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java index 7e79e1e..0724191 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java @@ -400,14 +400,14 @@ public void addPartitionColsToBatch(VectorizedRowBatch batch, Object[] partition } case INTERVAL_DAY_TIME: { - TimestampColumnVector tcv = (TimestampColumnVector) batch.cols[colIndex]; + IntervalDayTimeColumnVector icv = (IntervalDayTimeColumnVector) batch.cols[colIndex]; if (value == null) { - tcv.noNulls = false; - tcv.isNull[0] = true; - tcv.isRepeating = true; + icv.noNulls = false; + icv.isNull[0] = true; + icv.isRepeating = true; } else { - tcv.fill(((HiveIntervalDayTime) value).pisaTimestampUpdate(tcv.useScratchPisaTimestamp())); - tcv.isNull[0] = false; + icv.fill(((HiveIntervalDayTime) value)); + icv.isNull[0] = false; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java index 2b0068d..6225ade 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java @@ -44,7 +44,6 @@ public CastDecimalToTimestamp() { @Override protected void func(TimestampColumnVector outV, DecimalColumnVector inV, int i) { - Timestamp timestamp = TimestampWritable.decimalToTimestamp(inV.vector[i].getHiveDecimal()); - outV.set(i, timestamp); + outV.set(i, TimestampWritable.decimalToTimestamp(inV.vector[i].getHiveDecimal())); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToTimestamp.java index 39823fe..31d2f78 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToTimestamp.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; public class CastDoubleToTimestamp extends VectorExpression { private static final long serialVersionUID = 1L; @@ -37,9 +38,11 @@ public CastDoubleToTimestamp() { super(); } - private void setSecondsWithFractionalNanoseconds(TimestampColumnVector timestampColVector, + private void setDouble(TimestampColumnVector timestampColVector, double[] vector, int elementNum) { - timestampColVector.setTimestampSecondsWithFractionalNanoseconds(elementNum, vector[elementNum]); + TimestampWritable.setTimestampFromDouble( + timestampColVector.getScratchTimestamp(), vector[elementNum]); + timestampColVector.setFromScratchTimestamp(elementNum); } @Override @@ -66,7 +69,7 @@ public void evaluate(VectorizedRowBatch batch) { if (inputColVector.isRepeating) { //All must be selected otherwise size would be zero //Repeating property will not change. - setSecondsWithFractionalNanoseconds(outputColVector, vector, 0); + setDouble(outputColVector, vector, 0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; outputColVector.isRepeating = true; @@ -74,11 +77,11 @@ public void evaluate(VectorizedRowBatch batch) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - setSecondsWithFractionalNanoseconds(outputColVector, vector, i); + setDouble(outputColVector, vector, i); } } else { for(int i = 0; i != n; i++) { - setSecondsWithFractionalNanoseconds(outputColVector, vector, i); + setDouble(outputColVector, vector, i); } } outputColVector.isRepeating = false; @@ -86,12 +89,12 @@ public void evaluate(VectorizedRowBatch batch) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - setSecondsWithFractionalNanoseconds(outputColVector, vector, i); + setDouble(outputColVector, vector, i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - setSecondsWithFractionalNanoseconds(outputColVector, vector, i); + setDouble(outputColVector, vector, i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToTimestamp.java index d344d4d..a2ee52d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastLongToTimestamp.java @@ -20,8 +20,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; public class CastLongToTimestamp extends VectorExpression { private static final long serialVersionUID = 1L; @@ -40,7 +39,10 @@ public CastLongToTimestamp() { } private void setSeconds(TimestampColumnVector timestampColVector, long[] vector, int elementNum) { - timestampColVector.setTimestampSeconds(elementNum, vector[elementNum]); + TimestampWritable.setTimestampFromLong( + timestampColVector.getScratchTimestamp(), vector[elementNum], + /* intToTimestampInSeconds */ true); + timestampColVector.setFromScratchTimestamp(elementNum); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastMillisecondsLongToTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastMillisecondsLongToTimestamp.java index a0c947f..01c8810 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastMillisecondsLongToTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastMillisecondsLongToTimestamp.java @@ -20,8 +20,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; public class CastMillisecondsLongToTimestamp extends VectorExpression { private static final long serialVersionUID = 1L; @@ -39,6 +38,13 @@ public CastMillisecondsLongToTimestamp() { super(); } + private void setMilliseconds(TimestampColumnVector timestampColVector, long[] vector, int elementNum) { + TimestampWritable.setTimestampFromLong( + timestampColVector.getScratchTimestamp(), vector[elementNum], + /* intToTimestampInSeconds */ false); + timestampColVector.setFromScratchTimestamp(elementNum); + } + @Override public void evaluate(VectorizedRowBatch batch) { @@ -63,19 +69,19 @@ public void evaluate(VectorizedRowBatch batch) { if (inputColVector.isRepeating) { //All must be selected otherwise size would be zero //Repeating property will not change. - outputColVector.setTimestampMilliseconds(0, vector[0]); + setMilliseconds(outputColVector, vector, 0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; + outputIsNull[0] = inputIsNull[0]; outputColVector.isRepeating = true; } else if (inputColVector.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.setTimestampMilliseconds(i, vector[i]); + setMilliseconds(outputColVector, vector, i); } } else { for(int i = 0; i != n; i++) { - outputColVector.setTimestampMilliseconds(i, vector[i]); + setMilliseconds(outputColVector, vector, i); } } outputColVector.isRepeating = false; @@ -83,12 +89,12 @@ public void evaluate(VectorizedRowBatch batch) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.setTimestampMilliseconds(i, vector[i]); + setMilliseconds(outputColVector, vector, i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputColVector.setTimestampMilliseconds(i, vector[i]); + setMilliseconds(outputColVector, vector, i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToIntervalDayTime.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToIntervalDayTime.java index a3ddf9f..c8844c8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToIntervalDayTime.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToIntervalDayTime.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.serde.serdeConstants; @@ -56,7 +56,7 @@ public void evaluate(VectorizedRowBatch batch) { BytesColumnVector inV = (BytesColumnVector) batch.cols[inputColumn]; int[] sel = batch.selected; int n = batch.size; - TimestampColumnVector outV = (TimestampColumnVector) batch.cols[outputColumn]; + IntervalDayTimeColumnVector outV = (IntervalDayTimeColumnVector) batch.cols[outputColumn]; if (n == 0) { @@ -113,11 +113,11 @@ public void evaluate(VectorizedRowBatch batch) { } } - private void evaluate(TimestampColumnVector outV, BytesColumnVector inV, int i) { + private void evaluate(IntervalDayTimeColumnVector outV, BytesColumnVector inV, int i) { try { HiveIntervalDayTime interval = HiveIntervalDayTime.valueOf( new String(inV.vector[i], inV.start[i], inV.length[i], "UTF-8")); - outV.setEpochSecondsAndSignedNanos(i, interval.getTotalSeconds(), interval.getNanos()); + outV.set(i, interval); } catch (Exception e) { outV.setNullValue(i); outV.isNull[i] = true; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToBoolean.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToBoolean.java index 55b84b1..b8a58cd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToBoolean.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToBoolean.java @@ -41,8 +41,8 @@ public CastTimestampToBoolean() { } private int toBool(TimestampColumnVector timestampColVector, int index) { - return (timestampColVector.getEpochDay(index) != 0 || - timestampColVector.getNanoOfDay(index) != 0) ? 1 : 0; + return (timestampColVector.getTime(index) != 0 || + timestampColVector.getNanos(index) != 0) ? 1 : 0; } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDate.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDate.java index 00790b9..4e3e62c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDate.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDate.java @@ -44,6 +44,6 @@ public CastTimestampToDate(int inputColumn, int outputColumn) { @Override protected void func(LongColumnVector outV, TimestampColumnVector inV, int i) { - outV.vector[i] = DateWritable.millisToDays(inV.getTimestampMilliseconds(i)); + outV.vector[i] = DateWritable.millisToDays(inV.getTime(i)); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDecimal.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDecimal.java index aec104e..e5bfb15 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDecimal.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDecimal.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; /** * To be used to cast timestamp to decimal. @@ -39,11 +39,6 @@ public CastTimestampToDecimal(int inputColumn, int outputColumn) { @Override protected void func(DecimalColumnVector outV, TimestampColumnVector inV, int i) { - - // The BigDecimal class recommends not converting directly from double to BigDecimal, - // so we convert like the non-vectorized case and got through a string... - Double timestampDouble = inV.getTimestampSecondsWithFractionalNanos(i); - HiveDecimal result = HiveDecimal.create(timestampDouble.toString()); - outV.set(i, result); + outV.set(i, TimestampWritable.getHiveDecimal(inV.asScratchTimestamp(i))); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDouble.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDouble.java index f8737f9..c37b182 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDouble.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToDouble.java @@ -20,8 +20,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; public class CastTimestampToDouble extends VectorExpression { private static final long serialVersionUID = 1L; @@ -63,7 +62,7 @@ public void evaluate(VectorizedRowBatch batch) { if (inputColVector.isRepeating) { //All must be selected otherwise size would be zero //Repeating property will not change. - outputVector[0] = inputColVector.getTimestampSecondsWithFractionalNanos(0); + outputVector[0] = inputColVector.getTimestampAsDouble(0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; outputColVector.isRepeating = true; @@ -71,11 +70,11 @@ public void evaluate(VectorizedRowBatch batch) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = inputColVector.getTimestampSecondsWithFractionalNanos(i); + outputVector[i] = inputColVector.getTimestampAsDouble(i); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = inputColVector.getTimestampSecondsWithFractionalNanos(i); + outputVector[i] = inputColVector.getTimestampAsDouble(i); } } outputColVector.isRepeating = false; @@ -83,12 +82,12 @@ public void evaluate(VectorizedRowBatch batch) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = inputColVector.getTimestampSecondsWithFractionalNanos(i); + outputVector[i] = inputColVector.getTimestampAsDouble(i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputVector[i] = inputColVector.getTimestampSecondsWithFractionalNanos(i); + outputVector[i] = inputColVector.getTimestampAsDouble(i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToLong.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToLong.java index 4f53f5c..ba2e823 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToLong.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToLong.java @@ -64,19 +64,19 @@ public void evaluate(VectorizedRowBatch batch) { if (inputColVector.isRepeating) { //All must be selected otherwise size would be zero //Repeating property will not change. - outputVector[0] = inputColVector.getEpochSeconds(0); + outputVector[0] = inputColVector.getTimestampAsLong(0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. - outputIsNull[0] = inputIsNull[0]; + outputIsNull[0] = inputIsNull[0]; outputColVector.isRepeating = true; } else if (inputColVector.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = inputColVector.getEpochSeconds(i); + outputVector[i] = inputColVector.getTimestampAsLong(i); } } else { for(int i = 0; i != n; i++) { - outputVector[i] = inputColVector.getEpochSeconds(i); + outputVector[i] = inputColVector.getTimestampAsLong(i); } } outputColVector.isRepeating = false; @@ -84,12 +84,12 @@ public void evaluate(VectorizedRowBatch batch) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = inputColVector.getEpochSeconds(i); + outputVector[i] = inputColVector.getTimestampAsLong(i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputVector[i] = inputColVector.getEpochSeconds(i); + outputVector[i] = inputColVector.getTimestampAsLong(i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java index 24ee9bc..8a743f6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java @@ -24,13 +24,9 @@ import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.*; -import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hive.common.util.DateUtils; /** * Constant is represented as a vector with repeating values. @@ -44,7 +40,8 @@ private double doubleValue = 0; private byte[] bytesValue = null; private HiveDecimal decimalValue = null; - private PisaTimestamp timestampValue = null; + private Timestamp timestampValue = null; + private HiveIntervalDayTime intervalDayTimeValue = null; private boolean isNullValue = false; private ColumnVector.Type type; @@ -97,7 +94,7 @@ public ConstantVectorExpression(int outputColumn, Timestamp value) { } public ConstantVectorExpression(int outputColumn, HiveIntervalDayTime value) { - this(outputColumn, "timestamp"); + this(outputColumn, "interval_day_time"); setIntervalDayTimeValue(value); } @@ -165,6 +162,17 @@ private void evaluateTimestamp(VectorizedRowBatch vrg) { } } + private void evaluateIntervalDayTime(VectorizedRowBatch vrg) { + IntervalDayTimeColumnVector dcv = (IntervalDayTimeColumnVector) vrg.cols[outputColumn]; + dcv.isRepeating = true; + dcv.noNulls = !isNullValue; + if (!isNullValue) { + dcv.set(0, intervalDayTimeValue); + } else { + dcv.isNull[0] = true; + } + } + @Override public void evaluate(VectorizedRowBatch vrg) { switch (type) { @@ -183,6 +191,9 @@ public void evaluate(VectorizedRowBatch vrg) { case TIMESTAMP: evaluateTimestamp(vrg); break; + case INTERVAL_DAY_TIME: + evaluateIntervalDayTime(vrg); + break; } } @@ -225,16 +236,19 @@ public HiveDecimal getDecimalValue() { } public void setTimestampValue(Timestamp timestampValue) { - this.timestampValue = new PisaTimestamp(timestampValue); + this.timestampValue = timestampValue; } - public void setIntervalDayTimeValue(HiveIntervalDayTime intervalDayTimeValue) { - this.timestampValue = intervalDayTimeValue.pisaTimestampUpdate(new PisaTimestamp()); + public Timestamp getTimestampValue() { + return timestampValue; } + public void setIntervalDayTimeValue(HiveIntervalDayTime intervalDayTimeValue) { + this.intervalDayTimeValue = intervalDayTimeValue; + } - public PisaTimestamp getTimestampValue() { - return timestampValue; + public HiveIntervalDayTime getIntervalDayTimeValue() { + return intervalDayTimeValue; } public String getTypeString() { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateColumn.java index 8d2a186..fafacce 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateColumn.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateColumn.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; import org.apache.hadoop.hive.ql.exec.vector.*; @@ -26,7 +28,7 @@ import org.apache.hadoop.hive.serde2.io.DateWritable; // A type date (LongColumnVector storing epoch days) minus a type date produces a -// type interval_day_time (TimestampColumnVector storing nanosecond interval in 2 longs). +// type interval_day_time (IntervalDayTimeColumnVector storing nanosecond interval in 2 longs). public class DateColSubtractDateColumn extends VectorExpression { private static final long serialVersionUID = 1L; @@ -34,16 +36,16 @@ private int colNum1; private int colNum2; private int outputColumn; - private PisaTimestamp scratchPisaTimestamp1; - private PisaTimestamp scratchPisaTimestamp2; + private Timestamp scratchTimestamp1; + private Timestamp scratchTimestamp2; private DateTimeMath dtm = new DateTimeMath(); public DateColSubtractDateColumn(int colNum1, int colNum2, int outputColumn) { this.colNum1 = colNum1; this.colNum2 = colNum2; this.outputColumn = outputColumn; - scratchPisaTimestamp1 = new PisaTimestamp(); - scratchPisaTimestamp2 = new PisaTimestamp(); + scratchTimestamp1 = new Timestamp(0); + scratchTimestamp2 = new Timestamp(0); } public DateColSubtractDateColumn() { @@ -63,7 +65,7 @@ public void evaluate(VectorizedRowBatch batch) { LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; // Output is type interval_day_time. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; + IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; int n = batch.size; @@ -80,73 +82,69 @@ public void evaluate(VectorizedRowBatch batch) { || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; - // Handle nulls first + // Handle nulls first NullUtil.propagateNullsColCol( inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); + HiveIntervalDayTime resultIntervalDayTime = outputColVector.getScratchIntervalDayTime(); + /* Disregard nulls for processing. In other words, * the arithmetic operation is performed even if one or * more inputs are null. This is to improve speed by avoiding * conditional checks in the inner loop. */ if (inputColVector1.isRepeating && inputColVector2.isRepeating) { - outputColVector.subtract( - scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])), - scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])), - 0); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0])); + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0])); + dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(0); } else if (inputColVector1.isRepeating) { + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0])); if (batch.selectedInUse) { - scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])); for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.subtract( - scratchPisaTimestamp1, - scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } else { - scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])); for(int i = 0; i != n; i++) { - outputColVector.subtract( - scratchPisaTimestamp1, - scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } } else if (inputColVector2.isRepeating) { + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0])); if (batch.selectedInUse) { - scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])); for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.subtract( - scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - scratchPisaTimestamp2, - i); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } else { - scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])); for(int i = 0; i != n; i++) { - outputColVector.subtract( - scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - scratchPisaTimestamp2, - i); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } } else { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.subtract( - scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.subtract( - scratchPisaTimestamp1.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[i])), - scratchPisaTimestamp2.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.subtract(scratchTimestamp1, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } } @@ -157,7 +155,7 @@ public void evaluate(VectorizedRowBatch batch) { * in complex arithmetic expressions like col2 / (col1 - 1) * in the case when some col1 entries are null. */ - NullUtil.setNullDataEntriesTimestamp(outputColVector, batch.selectedInUse, sel, n); + NullUtil.setNullDataEntriesIntervalDayTime(outputColVector, batch.selectedInUse, sel, n); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java index 3ea9331..a9ca93c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateColSubtractDateScalar.java @@ -18,7 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; @@ -35,16 +36,17 @@ private static final long serialVersionUID = 1L; private int colNum; - private PisaTimestamp value; + private Timestamp value; private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; + private Timestamp scratchTimestamp1; private DateTimeMath dtm = new DateTimeMath(); public DateColSubtractDateScalar(int colNum, long value, int outputColumn) { this.colNum = colNum; - this.value = new PisaTimestamp().updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) value)); + this.value = new Timestamp(0); + this.value.setTime(DateWritable.daysToMillis((int) value)); this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); + scratchTimestamp1 = new Timestamp(0); } public DateColSubtractDateScalar() { @@ -60,8 +62,8 @@ public void evaluate(VectorizedRowBatch batch) { // Input #1 is type date (epochDays). LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum]; - // Output is type Timestamp. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; + // Output is type HiveIntervalDayTime. + IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; boolean[] inputIsNull = inputColVector1.isNull; @@ -77,45 +79,40 @@ public void evaluate(VectorizedRowBatch batch) { } if (inputColVector1.isRepeating) { - outputColVector.subtract( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])), - value, - 0); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[0])); + dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; } else if (inputColVector1.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.subtract( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])), - value, - i); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.subtract( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])), - value, - i); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } } else /* there are nulls */ { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.subtract( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])), - value, - i); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputColVector.subtract( - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector1[0])), - value, - i); + scratchTimestamp1.setTime(DateWritable.daysToMillis((int) vector1[i])); + dtm.subtract(scratchTimestamp1, value, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java index a8cabb8..59cf9da 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DateScalarSubtractDateColumn.java @@ -18,7 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; @@ -32,16 +33,17 @@ private static final long serialVersionUID = 1L; private int colNum; - private PisaTimestamp value; + private Timestamp value; private int outputColumn; - private PisaTimestamp scratchPisaTimestamp; + private Timestamp scratchTimestamp2; private DateTimeMath dtm = new DateTimeMath(); public DateScalarSubtractDateColumn(long value, int colNum, int outputColumn) { this.colNum = colNum; - this.value = new PisaTimestamp().updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) value)); + this.value = new Timestamp(0); + this.value.setTime(DateWritable.daysToMillis((int) value)); this.outputColumn = outputColumn; - scratchPisaTimestamp = new PisaTimestamp(); + scratchTimestamp2 = new Timestamp(0); } public DateScalarSubtractDateColumn() { @@ -62,8 +64,8 @@ public void evaluate(VectorizedRowBatch batch) { // Input #2 is type date (epochDays). LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum]; - // Output is type Timestamp. - TimestampColumnVector outputColVector = (TimestampColumnVector) batch.cols[outputColumn]; + // Output is type HiveIntervalDayTime. + IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; boolean[] inputIsNull = inputColVector2.isNull; @@ -80,46 +82,40 @@ public void evaluate(VectorizedRowBatch batch) { } if (inputColVector2.isRepeating) { - outputColVector.subtract( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[0])), - 0); - + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[0])); + dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(0); // Even if there are no nulls, we always copy over entry 0. Simplifies code. outputIsNull[0] = inputIsNull[0]; } else if (inputColVector2.noNulls) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.subtract( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } else { for(int i = 0; i != n; i++) { - outputColVector.subtract( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } } } else { /* there are nulls */ if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.subtract( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); outputIsNull[i] = inputIsNull[i]; } } else { for(int i = 0; i != n; i++) { - outputColVector.subtract( - value, - scratchPisaTimestamp.updateFromTimestampMilliseconds(DateWritable.daysToMillis((int) vector2[i])), - i); + scratchTimestamp2.setTime(DateWritable.daysToMillis((int) vector2[i])); + dtm.subtract(value, scratchTimestamp2, outputColVector.getScratchIntervalDayTime()); + outputColVector.setFromScratchIntervalDayTime(i); } System.arraycopy(inputIsNull, 0, outputIsNull, 0, n); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterTimestampColumnInList.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterTimestampColumnInList.java index 42e4984..25a276a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterTimestampColumnInList.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterTimestampColumnInList.java @@ -21,7 +21,6 @@ import java.sql.Timestamp; import java.util.HashSet; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Descriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -35,7 +34,7 @@ private Timestamp[] inListValues; // The set object containing the IN list. - private transient HashSet inSet; + private transient HashSet inSet; public FilterTimestampColumnInList() { super(); @@ -58,9 +57,9 @@ public void evaluate(VectorizedRowBatch batch) { } if (inSet == null) { - inSet = new HashSet(inListValues.length); + inSet = new HashSet(inListValues.length); for (Timestamp val : inListValues) { - inSet.add(new PisaTimestamp(val)); + inSet.add(val); } } @@ -74,16 +73,13 @@ public void evaluate(VectorizedRowBatch batch) { return; } - PisaTimestamp scratchTimestamp = new PisaTimestamp(); - if (inputColVector.noNulls) { if (inputColVector.isRepeating) { // All must be selected otherwise size would be zero // Repeating property will not change. - inputColVector.pisaTimestampUpdate(scratchTimestamp, 0); - if (!(inSet.contains(scratchTimestamp))) { + if (!(inSet.contains(inputColVector.asScratchTimestamp(0)))) { //Entire batch is filtered out. batch.size = 0; } @@ -91,8 +87,7 @@ public void evaluate(VectorizedRowBatch batch) { int newSize = 0; for(int j = 0; j != n; j++) { int i = sel[j]; - inputColVector.pisaTimestampUpdate(scratchTimestamp, i); - if (inSet.contains(scratchTimestamp)) { + if (inSet.contains(inputColVector.asScratchTimestamp(i))) { sel[newSize++] = i; } } @@ -100,8 +95,7 @@ public void evaluate(VectorizedRowBatch batch) { } else { int newSize = 0; for(int i = 0; i != n; i++) { - inputColVector.pisaTimestampUpdate(scratchTimestamp, i); - if (inSet.contains(scratchTimestamp)) { + if (inSet.contains(inputColVector.asScratchTimestamp(i))) { sel[newSize++] = i; } } @@ -116,8 +110,7 @@ public void evaluate(VectorizedRowBatch batch) { //All must be selected otherwise size would be zero //Repeating property will not change. if (!nullPos[0]) { - inputColVector.pisaTimestampUpdate(scratchTimestamp, 0); - if (!inSet.contains(scratchTimestamp)) { + if (!inSet.contains(inputColVector.asScratchTimestamp(0))) { //Entire batch is filtered out. batch.size = 0; @@ -130,8 +123,7 @@ public void evaluate(VectorizedRowBatch batch) { for(int j = 0; j != n; j++) { int i = sel[j]; if (!nullPos[i]) { - inputColVector.pisaTimestampUpdate(scratchTimestamp, i); - if (inSet.contains(scratchTimestamp)) { + if (inSet.contains(inputColVector.asScratchTimestamp(i))) { sel[newSize++] = i; } } @@ -143,8 +135,7 @@ public void evaluate(VectorizedRowBatch batch) { int newSize = 0; for(int i = 0; i != n; i++) { if (!nullPos[i]) { - inputColVector.pisaTimestampUpdate(scratchTimestamp, i); - if (inSet.contains(scratchTimestamp)) { + if (inSet.contains(inputColVector.asScratchTimestamp(i))) { sel[newSize++] = i; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnColumn.java index a6f8057..804923e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnColumn.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnColumn.java @@ -17,24 +17,123 @@ */ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** * Compute IF(expr1, expr2, expr3) for 3 input column expressions. * The first is always a boolean (LongColumnVector). * The second and third are long columns or long expression results. */ -public class IfExprIntervalDayTimeColumnColumn extends IfExprTimestampColumnColumnBase { +public class IfExprIntervalDayTimeColumnColumn extends VectorExpression { private static final long serialVersionUID = 1L; + private int arg1Column, arg2Column, arg3Column; + private int outputColumn; + public IfExprIntervalDayTimeColumnColumn(int arg1Column, int arg2Column, int arg3Column, int outputColumn) { - super(arg1Column, arg2Column, arg3Column, outputColumn); + this.arg1Column = arg1Column; + this.arg2Column = arg2Column; + this.arg3Column = arg3Column; + this.outputColumn = outputColumn; } public IfExprIntervalDayTimeColumnColumn() { super(); } + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; + IntervalDayTimeColumnVector arg2ColVector = (IntervalDayTimeColumnVector) batch.cols[arg2Column]; + IntervalDayTimeColumnVector arg3ColVector = (IntervalDayTimeColumnVector) batch.cols[arg3Column]; + IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = arg2ColVector.noNulls && arg3ColVector.noNulls; + outputColVector.isRepeating = false; // may override later + int n = batch.size; + long[] vector1 = arg1ColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + /* All the code paths below propagate nulls even if neither arg2 nor arg3 + * have nulls. This is to reduce the number of code paths and shorten the + * code, at the expense of maybe doing unnecessary work if neither input + * has nulls. This could be improved in the future by expanding the number + * of code paths. + */ + if (arg1ColVector.isRepeating) { + if (vector1[0] == 1) { + arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); + } else { + arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); + } + return; + } + + // extend any repeating values and noNulls indicator in the inputs + arg2ColVector.flatten(batch.selectedInUse, sel, n); + arg3ColVector.flatten(batch.selectedInUse, sel, n); + + if (arg1ColVector.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchIntervalDayTime(i) : arg3ColVector.asScratchIntervalDayTime(i)); + outputIsNull[i] = (vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchIntervalDayTime(i) : arg3ColVector.asScratchIntervalDayTime(i)); + outputIsNull[i] = (vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.asScratchIntervalDayTime(i) : arg3ColVector.asScratchIntervalDayTime(i)); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.asScratchIntervalDayTime(i) : arg3ColVector.asScratchIntervalDayTime(i)); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); + } + } + } + + // restore repeating and no nulls indicators + arg2ColVector.unFlatten(); + arg3ColVector.unFlatten(); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "interval_day_time"; + } @Override public VectorExpressionDescriptor.Descriptor getDescriptor() { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnScalar.java index 4beb50a..8face7d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeColumnScalar.java @@ -19,8 +19,10 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** * Compute IF(expr1, expr2, expr3) for 3 input column expressions. @@ -28,13 +30,20 @@ * The second is a column or non-constant expression result. * The third is a constant value. */ -public class IfExprIntervalDayTimeColumnScalar extends IfExprTimestampColumnScalarBase { +public class IfExprIntervalDayTimeColumnScalar extends VectorExpression { private static final long serialVersionUID = 1L; + private int arg1Column, arg2Column; + private HiveIntervalDayTime arg3Scalar; + private int outputColumn; + public IfExprIntervalDayTimeColumnScalar(int arg1Column, int arg2Column, HiveIntervalDayTime arg3Scalar, int outputColumn) { - super(arg1Column, arg2Column, arg3Scalar.pisaTimestampUpdate(new PisaTimestamp()), outputColumn); + this.arg1Column = arg1Column; + this.arg2Column = arg2Column; + this.arg3Scalar = arg3Scalar; + this.outputColumn = outputColumn; } public IfExprIntervalDayTimeColumnScalar() { @@ -42,6 +51,85 @@ public IfExprIntervalDayTimeColumnScalar() { } @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; + IntervalDayTimeColumnVector arg2ColVector = (IntervalDayTimeColumnVector) batch.cols[arg2Column]; + IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = arg2ColVector.noNulls; // nulls can only come from arg2 + outputColVector.isRepeating = false; // may override later + int n = batch.size; + long[] vector1 = arg1ColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (arg1ColVector.isRepeating) { + if (vector1[0] == 1) { + arg2ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); + } else { + outputColVector.fill(arg3Scalar); + } + return; + } + + // Extend any repeating values and noNulls indicator in the inputs to + // reduce the number of code paths needed below. + arg2ColVector.flatten(batch.selectedInUse, sel, n); + + if (arg1ColVector.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchIntervalDayTime(i) : arg3Scalar); + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchIntervalDayTime(i) : arg3Scalar); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.asScratchIntervalDayTime(i) : arg3Scalar); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.isNull[i] : false); + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.asScratchIntervalDayTime(i) : arg3Scalar); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2ColVector.isNull[i] : false); + } + } + } + + // restore repeating and no nulls indicators + arg2ColVector.unFlatten(); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "interval_day_time"; + } + + @Override public VectorExpressionDescriptor.Descriptor getDescriptor() { return (new VectorExpressionDescriptor.Builder()) .setMode( diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarColumn.java index 5463c7c..40f2e08 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarColumn.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarColumn.java @@ -19,8 +19,10 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** * Compute IF(expr1, expr2, expr3) for 3 input column expressions. @@ -28,13 +30,20 @@ * The second is a column or non-constant expression result. * The third is a constant value. */ -public class IfExprIntervalDayTimeScalarColumn extends IfExprTimestampScalarColumnBase { +public class IfExprIntervalDayTimeScalarColumn extends VectorExpression { private static final long serialVersionUID = 1L; + private int arg1Column, arg3Column; + private HiveIntervalDayTime arg2Scalar; + private int outputColumn; + public IfExprIntervalDayTimeScalarColumn(int arg1Column, HiveIntervalDayTime arg2Scalar, int arg3Column, int outputColumn) { - super(arg1Column, arg2Scalar.pisaTimestampUpdate(new PisaTimestamp()), arg3Column, outputColumn); + this.arg1Column = arg1Column; + this.arg2Scalar = arg2Scalar; + this.arg3Column = arg3Column; + this.outputColumn = outputColumn; } public IfExprIntervalDayTimeScalarColumn() { @@ -42,6 +51,87 @@ public IfExprIntervalDayTimeScalarColumn() { } @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; + IntervalDayTimeColumnVector arg3ColVector = (IntervalDayTimeColumnVector) batch.cols[arg3Column]; + IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = arg3ColVector.noNulls; // nulls can only come from arg3 column vector + outputColVector.isRepeating = false; // may override later + int n = batch.size; + long[] vector1 = arg1ColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (arg1ColVector.isRepeating) { + if (vector1[0] == 1) { + outputColVector.fill(arg2Scalar); + } else { + arg3ColVector.copySelected(batch.selectedInUse, sel, n, outputColVector); + } + return; + } + + // Extend any repeating values and noNulls indicator in the inputs to + // reduce the number of code paths needed below. + // This could be optimized in the future by having separate paths + // for when arg3ColVector is repeating or has no nulls. + arg3ColVector.flatten(batch.selectedInUse, sel, n); + + if (arg1ColVector.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchIntervalDayTime(i)); + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchIntervalDayTime(i)); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2Scalar : arg3ColVector.asScratchIntervalDayTime(i)); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + false : arg3ColVector.isNull[i]); + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2Scalar : arg3ColVector.asScratchIntervalDayTime(i)); + outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? + false : arg3ColVector.isNull[i]); + } + } + } + + // restore repeating and no nulls indicators + arg3ColVector.unFlatten(); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "interval_day_time"; + } + + @Override public VectorExpressionDescriptor.Descriptor getDescriptor() { return (new VectorExpressionDescriptor.Builder()) .setMode( diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarScalar.java index af2e0c0..43676dd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprIntervalDayTimeScalarScalar.java @@ -18,9 +18,13 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import java.util.Arrays; + import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** * Compute IF(expr1, expr2, expr3) for 3 input expressions. @@ -28,13 +32,21 @@ * The second is a constant value. * The third is a constant value. */ -public class IfExprIntervalDayTimeScalarScalar extends IfExprTimestampScalarScalarBase { +public class IfExprIntervalDayTimeScalarScalar extends VectorExpression { private static final long serialVersionUID = 1L; + private int arg1Column; + private HiveIntervalDayTime arg2Scalar; + private HiveIntervalDayTime arg3Scalar; + private int outputColumn; + public IfExprIntervalDayTimeScalarScalar(int arg1Column, HiveIntervalDayTime arg2Scalar, HiveIntervalDayTime arg3Scalar, int outputColumn) { - super(arg1Column, arg2Scalar.pisaTimestampUpdate(new PisaTimestamp()), arg3Scalar.pisaTimestampUpdate(new PisaTimestamp()), outputColumn); + this.arg1Column = arg1Column; + this.arg2Scalar = arg2Scalar; + this.arg3Scalar = arg3Scalar; + this.outputColumn = outputColumn; } public IfExprIntervalDayTimeScalarScalar() { @@ -42,6 +54,72 @@ public IfExprIntervalDayTimeScalarScalar() { } @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; + IntervalDayTimeColumnVector outputColVector = (IntervalDayTimeColumnVector) batch.cols[outputColumn]; + int[] sel = batch.selected; + boolean[] outputIsNull = outputColVector.isNull; + outputColVector.noNulls = false; // output is a scalar which we know is non null + outputColVector.isRepeating = false; // may override later + int n = batch.size; + long[] vector1 = arg1ColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + if (arg1ColVector.isRepeating) { + if (vector1[0] == 1) { + outputColVector.fill(arg2Scalar); + } else { + outputColVector.fill(arg3Scalar); + } + } else if (arg1ColVector.noNulls) { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3Scalar); + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3Scalar); + } + } + } else /* there are nulls */ { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2Scalar : arg3Scalar); + outputIsNull[i] = false; + } + } else { + for(int i = 0; i != n; i++) { + outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? + arg2Scalar : arg3Scalar); + } + Arrays.fill(outputIsNull, 0, n, false); + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String getOutputType() { + return "timestamp"; + } + + @Override public VectorExpressionDescriptor.Descriptor getDescriptor() { return (new VectorExpressionDescriptor.Builder()) .setMode( diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnColumnBase.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnColumnBase.java index d3dd67d..8441863 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnColumnBase.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnColumnBase.java @@ -89,13 +89,13 @@ public void evaluate(VectorizedRowBatch batch) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchPisaTimestamp(i) : arg3ColVector.asScratchPisaTimestamp(i)); + outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchTimestamp(i) : arg3ColVector.asScratchTimestamp(i)); outputIsNull[i] = (vector1[i] == 1 ? arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchPisaTimestamp(i) : arg3ColVector.asScratchPisaTimestamp(i)); + outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchTimestamp(i) : arg3ColVector.asScratchTimestamp(i)); outputIsNull[i] = (vector1[i] == 1 ? arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); } @@ -105,14 +105,14 @@ public void evaluate(VectorizedRowBatch batch) { for(int j = 0; j != n; j++) { int i = sel[j]; outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? - arg2ColVector.asScratchPisaTimestamp(i) : arg3ColVector.asScratchPisaTimestamp(i)); + arg2ColVector.asScratchTimestamp(i) : arg3ColVector.asScratchTimestamp(i)); outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); } } else { for(int i = 0; i != n; i++) { outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? - arg2ColVector.asScratchPisaTimestamp(i) : arg3ColVector.asScratchPisaTimestamp(i)); + arg2ColVector.asScratchTimestamp(i) : arg3ColVector.asScratchTimestamp(i)); outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? arg2ColVector.isNull[i] : arg3ColVector.isNull[i]); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalar.java index 0660038..ae997e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalar.java @@ -20,7 +20,6 @@ import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; /** @@ -35,7 +34,7 @@ public IfExprTimestampColumnScalar(int arg1Column, int arg2Column, Timestamp arg3Scalar, int outputColumn) { - super(arg1Column, arg2Column, new PisaTimestamp(arg3Scalar), outputColumn); + super(arg1Column, arg2Column, arg3Scalar, outputColumn); } public IfExprTimestampColumnScalar() { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalarBase.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalarBase.java index 8aaad3f..6b87ff2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalarBase.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampColumnScalarBase.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -38,10 +38,10 @@ private static final long serialVersionUID = 1L; private int arg1Column, arg2Column; - private PisaTimestamp arg3Scalar; + private Timestamp arg3Scalar; private int outputColumn; - public IfExprTimestampColumnScalarBase(int arg1Column, int arg2Column, PisaTimestamp arg3Scalar, + public IfExprTimestampColumnScalarBase(int arg1Column, int arg2Column, Timestamp arg3Scalar, int outputColumn) { this.arg1Column = arg1Column; this.arg2Column = arg2Column; @@ -91,11 +91,11 @@ public void evaluate(VectorizedRowBatch batch) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchPisaTimestamp(i) : arg3Scalar); + outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchTimestamp(i) : arg3Scalar); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchPisaTimestamp(i) : arg3Scalar); + outputColVector.set(i, vector1[i] == 1 ? arg2ColVector.asScratchTimestamp(i) : arg3Scalar); } } } else /* there are nulls */ { @@ -103,14 +103,14 @@ public void evaluate(VectorizedRowBatch batch) { for(int j = 0; j != n; j++) { int i = sel[j]; outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? - arg2ColVector.asScratchPisaTimestamp(i) : arg3Scalar); + arg2ColVector.asScratchTimestamp(i) : arg3Scalar); outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? arg2ColVector.isNull[i] : false); } } else { for(int i = 0; i != n; i++) { outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? - arg2ColVector.asScratchPisaTimestamp(i) : arg3Scalar); + arg2ColVector.asScratchTimestamp(i) : arg3Scalar); outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? arg2ColVector.isNull[i] : false); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumn.java index 7f618cb..3d53df1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumn.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumn.java @@ -20,7 +20,6 @@ import java.sql.Timestamp; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; /** @@ -35,7 +34,7 @@ public IfExprTimestampScalarColumn(int arg1Column, Timestamp arg2Scalar, int arg3Column, int outputColumn) { - super(arg1Column, new PisaTimestamp(arg2Scalar), arg3Column, outputColumn); + super(arg1Column, arg2Scalar, arg3Column, outputColumn); } public IfExprTimestampScalarColumn() { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumnBase.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumnBase.java index 84d7655..2162f17 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumnBase.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarColumnBase.java @@ -18,7 +18,8 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import java.sql.Timestamp; + import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; @@ -36,10 +37,10 @@ private static final long serialVersionUID = 1L; private int arg1Column, arg3Column; - private PisaTimestamp arg2Scalar; + private Timestamp arg2Scalar; private int outputColumn; - public IfExprTimestampScalarColumnBase(int arg1Column, PisaTimestamp arg2Scalar, int arg3Column, + public IfExprTimestampScalarColumnBase(int arg1Column, Timestamp arg2Scalar, int arg3Column, int outputColumn) { this.arg1Column = arg1Column; this.arg2Scalar = arg2Scalar; @@ -91,11 +92,11 @@ public void evaluate(VectorizedRowBatch batch) { if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchPisaTimestamp(i)); + outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchTimestamp(i)); } } else { for(int i = 0; i != n; i++) { - outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchPisaTimestamp(i)); + outputColVector.set(i, vector1[i] == 1 ? arg2Scalar : arg3ColVector.asScratchTimestamp(i)); } } } else /* there are nulls */ { @@ -103,14 +104,14 @@ public void evaluate(VectorizedRowBatch batch) { for(int j = 0; j != n; j++) { int i = sel[j]; outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? - arg2Scalar : arg3ColVector.asScratchPisaTimestamp(i)); + arg2Scalar : arg3ColVector.asScratchTimestamp(i)); outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? false : arg3ColVector.isNull[i]); } } else { for(int i = 0; i != n; i++) { outputColVector.set(i, !arg1ColVector.isNull[i] && vector1[i] == 1 ? - arg2Scalar : arg3ColVector.asScratchPisaTimestamp(i)); + arg2Scalar : arg3ColVector.asScratchTimestamp(i)); outputIsNull[i] = (!arg1ColVector.isNull[i] && vector1[i] == 1 ? false : arg3ColVector.isNull[i]); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalar.java index 5286ea3..cd00d3a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalar.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import java.sql.Timestamp; @@ -35,7 +34,7 @@ public IfExprTimestampScalarScalar(int arg1Column, Timestamp arg2Scalar, Timestamp arg3Scalar, int outputColumn) { - super(arg1Column, new PisaTimestamp(arg2Scalar), new PisaTimestamp(arg3Scalar), outputColumn); + super(arg1Column, arg2Scalar, arg3Scalar, outputColumn); } public IfExprTimestampScalarScalar() { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalarBase.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalarBase.java index 1aeabfc..707f574 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalarBase.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprTimestampScalarScalarBase.java @@ -18,11 +18,12 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +import java.sql.Timestamp; import java.util.Arrays; /** @@ -36,11 +37,11 @@ private static final long serialVersionUID = 1L; private int arg1Column; - private PisaTimestamp arg2Scalar; - private PisaTimestamp arg3Scalar; + private Timestamp arg2Scalar; + private Timestamp arg3Scalar; private int outputColumn; - public IfExprTimestampScalarScalarBase(int arg1Column, PisaTimestamp arg2Scalar, PisaTimestamp arg3Scalar, + public IfExprTimestampScalarScalarBase(int arg1Column, Timestamp arg2Scalar, Timestamp arg3Scalar, int outputColumn) { this.arg1Column = arg1Column; this.arg2Scalar = arg2Scalar; @@ -116,8 +117,4 @@ public int getOutputColumn() { public String getOutputType() { return "timestamp"; } - - public int getArg1Column() { - return arg1Column; - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java index 3c6824d..eb493bf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.IntervalDayTimeColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; @@ -107,6 +108,31 @@ public static void setNullDataEntriesTimestamp( } } + /** + * Set the data value for all NULL entries to the designated NULL_VALUE. + */ + public static void setNullDataEntriesIntervalDayTime( + IntervalDayTimeColumnVector v, boolean selectedInUse, int[] sel, int n) { + if (v.noNulls) { + return; + } else if (v.isRepeating && v.isNull[0]) { + v.setNullValue(0); + } else if (selectedInUse) { + for (int j = 0; j != n; j++) { + int i = sel[j]; + if(v.isNull[i]) { + v.setNullValue(i); + } + } + } else { + for (int i = 0; i != n; i++) { + if(v.isNull[i]) { + v.setNullValue(i); + } + } + } + } + // for use by Column-Scalar and Scalar-Column arithmetic for null propagation public static void setNullOutputEntriesColScalar( ColumnVector v, boolean selectedInUse, int[] sel, int n) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampColumnInList.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampColumnInList.java index 2d7d0c2..bc09a3a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampColumnInList.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/TimestampColumnInList.java @@ -21,7 +21,6 @@ import java.sql.Timestamp; import java.util.HashSet; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Descriptor; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; @@ -36,11 +35,8 @@ private Timestamp[] inListValues; private int outputColumn; - private transient PisaTimestamp scratchTimestamp; - - // The set object containing the IN list. - private transient HashSet inSet; + private transient HashSet inSet; public TimestampColumnInList() { super(); @@ -64,11 +60,10 @@ public void evaluate(VectorizedRowBatch batch) { } if (inSet == null) { - inSet = new HashSet(inListValues.length); + inSet = new HashSet(inListValues.length); for (Timestamp val : inListValues) { - inSet.add(new PisaTimestamp(val)); + inSet.add(val); } - scratchTimestamp = new PisaTimestamp(); } TimestampColumnVector inputColVector = (TimestampColumnVector) batch.cols[inputCol]; @@ -91,19 +86,16 @@ public void evaluate(VectorizedRowBatch batch) { // All must be selected otherwise size would be zero // Repeating property will not change. - inputColVector.pisaTimestampUpdate(scratchTimestamp, 0); - outputVector[0] = inSet.contains(scratchTimestamp) ? 1 : 0; + outputVector[0] = inSet.contains(inputColVector.asScratchTimestamp(0)) ? 1 : 0; outputColVector.isRepeating = true; } else if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - inputColVector.pisaTimestampUpdate(scratchTimestamp, i); - outputVector[i] = inSet.contains(scratchTimestamp) ? 1 : 0; + outputVector[i] = inSet.contains(inputColVector.asScratchTimestamp(i)) ? 1 : 0; } } else { for(int i = 0; i != n; i++) { - inputColVector.pisaTimestampUpdate(scratchTimestamp, i); - outputVector[i] = inSet.contains(scratchTimestamp) ? 1 : 0; + outputVector[i] = inSet.contains(inputColVector.asScratchTimestamp(i)) ? 1 : 0; } } } else { @@ -112,8 +104,7 @@ public void evaluate(VectorizedRowBatch batch) { //All must be selected otherwise size would be zero //Repeating property will not change. if (!nullPos[0]) { - inputColVector.pisaTimestampUpdate(scratchTimestamp, 0); - outputVector[0] = inSet.contains(scratchTimestamp) ? 1 : 0; + outputVector[0] = inSet.contains(inputColVector.asScratchTimestamp(0)) ? 1 : 0; outNulls[0] = false; } else { outNulls[0] = true; @@ -124,16 +115,14 @@ public void evaluate(VectorizedRowBatch batch) { int i = sel[j]; outNulls[i] = nullPos[i]; if (!nullPos[i]) { - inputColVector.pisaTimestampUpdate(scratchTimestamp, i); - outputVector[i] = inSet.contains(scratchTimestamp) ? 1 : 0; + outputVector[i] = inSet.contains(inputColVector.asScratchTimestamp(i)) ? 1 : 0; } } } else { System.arraycopy(nullPos, 0, outNulls, 0, n); for(int i = 0; i != n; i++) { if (!nullPos[i]) { - inputColVector.pisaTimestampUpdate(scratchTimestamp, i); - outputVector[i] = inSet.contains(scratchTimestamp) ? 1 : 0; + outputVector[i] = inSet.contains(inputColVector.asScratchTimestamp(i)) ? 1 : 0; } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java index 326bfb9..85dacd7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java @@ -21,10 +21,11 @@ import java.sql.Timestamp; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.common.type.PisaTimestamp; +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -42,7 +43,8 @@ Object writeValue(HiveDecimal value) throws HiveException; Object writeValue(TimestampWritable value) throws HiveException; Object writeValue(Timestamp value) throws HiveException; - Object writeValue(PisaTimestamp value) throws HiveException; + Object writeValue(HiveIntervalDayTimeWritable value) throws HiveException; + Object writeValue(HiveIntervalDayTime value) throws HiveException; Object setValue(Object row, ColumnVector column, int columnRow) throws HiveException; Object initValue(Object ost) throws HiveException; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java index 9a1d7f3..c20bc68 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.common.type.PisaTimestamp; import org.apache.hadoop.hive.ql.exec.vector.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -188,17 +187,39 @@ public Object setValue(Object field, TimestampWritable value) throws HiveExcepti } /** - * The base implementation must be overridden by the PisaTimestamp specialization + * The base implementation must be overridden by the Timestamp specialization + */ + public Object setValue(Object field, Timestamp value) throws HiveException { + throw new HiveException("Internal error: should not reach here"); + } + + /** + * The base implementation must be overridden by the HiveIntervalDayTime specialization */ @Override - public Object writeValue(PisaTimestamp value) throws HiveException { + public Object writeValue(HiveIntervalDayTimeWritable value) throws HiveException { throw new HiveException("Internal error: should not reach here"); } /** - * The base implementation must be overridden by the Timestamp specialization + * The base implementation must be overridden by the HiveIntervalDayTime specialization */ - public Object setValue(Object field, Timestamp value) throws HiveException { + @Override + public Object writeValue(HiveIntervalDayTime value) throws HiveException { + throw new HiveException("Internal error: should not reach here"); + } + + /** + * The base implementation must be overridden by the HiveIntervalDayTime specialization + */ + public Object setValue(Object field, HiveIntervalDayTimeWritable value) throws HiveException { + throw new HiveException("Internal error: should not reach here"); + } + + /** + * The base implementation must be overridden by the HiveIntervalDayTime specialization + */ + public Object setValue(Object field, HiveIntervalDayTime value) throws HiveException { throw new HiveException("Internal error: should not reach here"); } } @@ -465,6 +486,66 @@ public Object setValue(Object field, ColumnVector column, int row) throws HiveEx } } + /** + * Specialized writer for IntervalDayTimeColumnVector. Will throw cast exception + * if the wrong vector column is used. + */ + private static abstract class VectorExpressionWriterIntervalDayTime extends VectorExpressionWriterBase { + @Override + public Object writeValue(ColumnVector column, int row) throws HiveException { + IntervalDayTimeColumnVector dcv = (IntervalDayTimeColumnVector) column; + HiveIntervalDayTimeWritable intervalDayTimeWritable = (HiveIntervalDayTimeWritable) dcv.getScratchWritable(); + if (intervalDayTimeWritable == null) { + intervalDayTimeWritable = new HiveIntervalDayTimeWritable(); + dcv.setScratchWritable(intervalDayTimeWritable); + } + if (dcv.noNulls && !dcv.isRepeating) { + return writeValue(TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, row, intervalDayTimeWritable)); + } else if (dcv.noNulls && dcv.isRepeating) { + return writeValue(TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, 0, intervalDayTimeWritable)); + } else if (!dcv.noNulls && !dcv.isRepeating && !dcv.isNull[row]) { + return writeValue(TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, row, intervalDayTimeWritable)); + } else if (!dcv.noNulls && dcv.isRepeating && !dcv.isNull[0]) { + return writeValue(TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, 0, intervalDayTimeWritable)); + } else if (!dcv.noNulls && dcv.isRepeating && dcv.isNull[0]) { + return null; + } else if (!dcv.noNulls && !dcv.isRepeating && dcv.isNull[row]) { + return null; + } + throw new HiveException( + String.format( + "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b", + row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0])); + } + + @Override + public Object setValue(Object field, ColumnVector column, int row) throws HiveException { + IntervalDayTimeColumnVector dcv = (IntervalDayTimeColumnVector) column; + HiveIntervalDayTimeWritable intervalDayTimeWritable = (HiveIntervalDayTimeWritable) dcv.getScratchWritable(); + if (intervalDayTimeWritable == null) { + intervalDayTimeWritable = new HiveIntervalDayTimeWritable(); + dcv.setScratchWritable(intervalDayTimeWritable); + } + if (dcv.noNulls && !dcv.isRepeating) { + return setValue(field, TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, row, intervalDayTimeWritable)); + } else if (dcv.noNulls && dcv.isRepeating) { + return setValue(field, TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, 0, intervalDayTimeWritable)); + } else if (!dcv.noNulls && !dcv.isRepeating && !dcv.isNull[row]) { + return setValue(field, TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, row, intervalDayTimeWritable)); + } else if (!dcv.noNulls && !dcv.isRepeating && dcv.isNull[row]) { + return null; + } else if (!dcv.noNulls && dcv.isRepeating && !dcv.isNull[0]) { + return setValue(field, TimestampUtils.intervalDayTimeColumnVectorWritable(dcv, 0, intervalDayTimeWritable)); + } else if (!dcv.noNulls && dcv.isRepeating && dcv.isNull[0]) { + return null; + } + throw new HiveException( + String.format( + "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b", + row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0])); + } + } + /** * Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc) */ @@ -697,8 +778,13 @@ public Object writeValue(Timestamp value) throws HiveException { } @Override - public Object writeValue(PisaTimestamp value) throws HiveException { - return ((SettableTimestampObjectInspector) this.objectInspector).set(obj, value.asScratchTimestamp()); + public Object writeValue(HiveIntervalDayTimeWritable value) throws HiveException { + return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, value); + } + + @Override + public Object writeValue(HiveIntervalDayTime value) throws HiveException { + return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, value); } @Override @@ -766,53 +852,45 @@ public Object initValue(Object ignored) { private static VectorExpressionWriter genVectorExpressionWritableIntervalDayTime( SettableHiveIntervalDayTimeObjectInspector fieldObjInspector) throws HiveException { - return new VectorExpressionWriterTimestamp() { + return new VectorExpressionWriterIntervalDayTime() { private Object obj; private HiveIntervalDayTime interval; - private PisaTimestamp pisaTimestamp; public VectorExpressionWriter init(SettableHiveIntervalDayTimeObjectInspector objInspector) throws HiveException { super.init(objInspector); interval = new HiveIntervalDayTime(); obj = initValue(null); - pisaTimestamp = new PisaTimestamp(); return this; } @Override - public Object writeValue(TimestampWritable value) throws HiveException { - interval.set(pisaTimestamp.updateFromTimestamp(value.getTimestamp())); + public Object writeValue(HiveIntervalDayTimeWritable value) throws HiveException { + interval.set(value.getHiveIntervalDayTime()); return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, interval); } @Override - public Object writeValue(Timestamp value) throws HiveException { - interval.set(pisaTimestamp.updateFromTimestamp(value)); - return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, interval); - } - - @Override - public Object writeValue(PisaTimestamp value) throws HiveException { + public Object writeValue(HiveIntervalDayTime value) throws HiveException { interval.set(value); return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(obj, interval); } @Override - public Object setValue(Object field, TimestampWritable value) { + public Object setValue(Object field, HiveIntervalDayTimeWritable value) { if (null == field) { field = initValue(null); } - interval.set(pisaTimestamp.updateFromTimestamp(value.getTimestamp())); + interval.set(value.getHiveIntervalDayTime()); return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(field, interval); } @Override - public Object setValue(Object field, Timestamp value) { + public Object setValue(Object field, HiveIntervalDayTime value) { if (null == field) { field = initValue(null); } - interval.set(pisaTimestamp.updateFromTimestamp(value)); + interval.set(value); return ((SettableHiveIntervalDayTimeObjectInspector) this.objectInspector).set(field, interval); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java index 9f5c793..05dd93e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColCol.java @@ -166,7 +166,7 @@ public void evaluate(VectorizedRowBatch batch) { protected byte[] evaluateTimestamp(ColumnVector columnVector, int index, long numDays) { TimestampColumnVector tcv = (TimestampColumnVector) columnVector; - calendar.setTimeInMillis(tcv.getTimestampMilliseconds(index)); + calendar.setTimeInMillis(tcv.getTime(index)); if (isPositive) { calendar.add(Calendar.DATE, (int) numDays); } else { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java index 6390ecd..59ca61e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateAddColScalar.java @@ -210,7 +210,7 @@ public void evaluate(VectorizedRowBatch batch) { protected byte[] evaluateTimestamp(ColumnVector columnVector, int index) { TimestampColumnVector tcv = (TimestampColumnVector) columnVector; - calendar.setTimeInMillis(tcv.getTimestampMilliseconds(index)); + calendar.setTimeInMillis(tcv.getTime(index)); if (isPositive) { calendar.add(Calendar.DATE, numDays); } else { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java index b22c31f..4edf558 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColCol.java @@ -275,7 +275,7 @@ public void copySelected( output.isRepeating = true; if (!input.isNull[0]) { - date.setTime(input.getTimestampMilliseconds(0)); + date.setTime(input.getTime(0)); output.vector[0] = DateWritable.dateToDays(date); } return; @@ -288,12 +288,12 @@ public void copySelected( if (selectedInUse) { for (int j = 0; j < size; j++) { int i = sel[j]; - date.setTime(input.getTimestampMilliseconds(i)); + date.setTime(input.getTime(i)); output.vector[i] = DateWritable.dateToDays(date); } } else { for (int i = 0; i < size; i++) { - date.setTime(input.getTimestampMilliseconds(i)); + date.setTime(input.getTime(i)); output.vector[i] = DateWritable.dateToDays(date); } } @@ -312,14 +312,14 @@ public void copySelected( for (int j = 0; j < size; j++) { int i = sel[j]; if (!input.isNull[i]) { - date.setTime(input.getTimestampMilliseconds(i)); + date.setTime(input.getTime(i)); output.vector[i] = DateWritable.dateToDays(date); } } } else { for (int i = 0; i < size; i++) { if (!input.isNull[i]) { - date.setTime(input.getTimestampMilliseconds(i)); + date.setTime(input.getTime(i)); output.vector[i] = DateWritable.dateToDays(date); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java index ab71b47..71b3887 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffColScalar.java @@ -238,7 +238,7 @@ public void evaluate(VectorizedRowBatch batch) { protected int evaluateTimestamp(ColumnVector columnVector, int index) { TimestampColumnVector tcv = (TimestampColumnVector) columnVector; - date.setTime(tcv.getTimestampMilliseconds(index)); + date.setTime(tcv.getTime(index)); return DateWritable.dateToDays(date) - baseDate; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java index dea5444..c733bc9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateDiffScalarCol.java @@ -237,7 +237,7 @@ public void evaluate(VectorizedRowBatch batch) { protected int evaluateTimestamp(ColumnVector columnVector, int index) { TimestampColumnVector tcv = (TimestampColumnVector) columnVector; - date.setTime(tcv.getTimestampMilliseconds(index)); + date.setTime(tcv.getTime(index)); return baseDate - DateWritable.dateToDays(date); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java index c29e22e..cde0be4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateTimestamp.java @@ -45,7 +45,7 @@ public VectorUDFDateTimestamp(int inputColumn, int outputColumn) { protected void func(BytesColumnVector outV, TimestampColumnVector inV, int i) { switch (inputTypes[0]) { case TIMESTAMP: - date.setTime(inV.getTimestampMilliseconds(i)); + date.setTime(inV.getTime(i)); break; default: diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampTimestamp.java index e4a31ca..6c33274 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampTimestamp.java @@ -18,10 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import java.sql.Timestamp; - import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; -import org.apache.hadoop.hive.serde2.io.DateWritable; /** * Return Unix Timestamp. @@ -33,7 +30,7 @@ @Override protected long getTimestampField(TimestampColumnVector timestampColVector, int elementNum) { - return timestampColVector.getTimestampSeconds(elementNum); + return timestampColVector.getTime(elementNum) / 1000; } public VectorUDFUnixTimeStampTimestamp(int colNum, int outputColumn) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java index 5c8db41..da684d3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -146,7 +147,8 @@ public void aggregateInputSelection( if (inputColVector.isRepeating) { iterateNoNullsRepeatingWithAggregationSelection( aggregationBufferSets, bufferIndex, - inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize); + TimestampWritable.getDouble(inputColVector.asScratchTimestamp(0)), + batchSize); } else { if (batch.selectedInUse) { iterateNoNullsSelectionWithAggregationSelection( @@ -163,11 +165,11 @@ public void aggregateInputSelection( if (batch.selectedInUse) { iterateHasNullsRepeatingSelectionWithAggregationSelection( aggregationBufferSets, bufferIndex, - inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize, batch.selected, inputColVector.isNull); + TimestampWritable.getDouble(inputColVector.asScratchTimestamp(0)), batchSize, batch.selected, inputColVector.isNull); } else { iterateHasNullsRepeatingWithAggregationSelection( aggregationBufferSets, bufferIndex, - inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize, inputColVector.isNull); + TimestampWritable.getDouble(inputColVector.asScratchTimestamp(0)), batchSize, inputColVector.isNull); } } else { if (batch.selectedInUse) { @@ -210,7 +212,8 @@ private void iterateNoNullsSelectionWithAggregationSelection( aggregationBufferSets, bufferIndex, i); - myagg.sumValue(inputColVector.getTimestampSecondsWithFractionalNanos(selection[i])); + myagg.sumValue( + TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selection[i]))); } } @@ -224,7 +227,7 @@ private void iterateNoNullsWithAggregationSelection( aggregationBufferSets, bufferIndex, i); - myagg.sumValue(inputColVector.getTimestampSecondsWithFractionalNanos(i)); + myagg.sumValue(TimestampWritable.getDouble(inputColVector.asScratchTimestamp(i))); } } @@ -281,7 +284,7 @@ private void iterateHasNullsSelectionWithAggregationSelection( aggregationBufferSets, bufferIndex, j); - myagg.sumValue(inputColVector.getTimestampSecondsWithFractionalNanos(i)); + myagg.sumValue(TimestampWritable.getDouble(inputColVector.asScratchTimestamp(i))); } } } @@ -296,10 +299,10 @@ private void iterateHasNullsWithAggregationSelection( for (int i=0; i < batchSize; ++i) { if (!isNull[i]) { Aggregation myagg = getCurrentAggregationBuffer( - aggregationBufferSets, + aggregationBufferSets, bufferIndex, i); - myagg.sumValue(inputColVector.getTimestampSecondsWithFractionalNanos(i)); + myagg.sumValue(TimestampWritable.getDouble(inputColVector.asScratchTimestamp(i))); } } } @@ -328,7 +331,7 @@ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) myagg.sum = 0; myagg.count = 0; } - myagg.sum += inputColVector.getTimestampSecondsWithFractionalNanos(0)*batchSize; + myagg.sum += TimestampWritable.getDouble(inputColVector.asScratchTimestamp(0))*batchSize; myagg.count += batchSize; } return; @@ -358,7 +361,7 @@ private void iterateSelectionHasNulls( for (int j=0; j< batchSize; ++j) { int i = selected[j]; if (!isNull[i]) { - double value = inputColVector.getTimestampSecondsWithFractionalNanos(i); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(i)); if (myagg.isNull) { myagg.isNull = false; myagg.sum = 0; @@ -381,24 +384,24 @@ private void iterateSelectionNoNulls( myagg.sum = 0; myagg.count = 0; } - + for (int i=0; i< batchSize; ++i) { - double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selected[i])); myagg.sum += value; myagg.count += 1; } } private void iterateNoSelectionHasNulls( - Aggregation myagg, - TimestampColumnVector inputColVector, + Aggregation myagg, + TimestampColumnVector inputColVector, int batchSize, boolean[] isNull) { - + for(int i=0;i 1) { @@ -322,7 +323,7 @@ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) if (inputColVector.isRepeating) { if (inputColVector.noNulls) { - iterateRepeatingNoNulls(myagg, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize); + iterateRepeatingNoNulls(myagg, TimestampWritable.getDouble(inputColVector.asScratchTimestamp(0)), batchSize); } } else if (!batch.selectedInUse && inputColVector.noNulls) { @@ -377,7 +378,7 @@ private void iterateSelectionHasNulls( for (int j=0; j< batchSize; ++j) { int i = selected[j]; if (!isNull[i]) { - double value = inputColVector.getTimestampSecondsWithFractionalNanos(i); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(i)); if (myagg.isNull) { myagg.init (); } @@ -401,7 +402,7 @@ private void iterateSelectionNoNulls( myagg.init (); } - double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[0]); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selected[0])); myagg.sum += value; myagg.count += 1; if(myagg.count > 1) { @@ -412,7 +413,7 @@ private void iterateSelectionNoNulls( // i=0 was pulled out to remove the count > 1 check in the loop // for (int i=1; i< batchSize; ++i) { - value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]); + value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selected[i])); myagg.sum += value; myagg.count += 1; double t = myagg.count*value - myagg.sum; @@ -428,7 +429,7 @@ private void iterateNoSelectionHasNulls( for(int i=0;i 1 check for (int i=1; i 1) { @@ -323,7 +324,7 @@ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) if (inputColVector.isRepeating) { if (inputColVector.noNulls) { - iterateRepeatingNoNulls(myagg, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize); + iterateRepeatingNoNulls(myagg, TimestampWritable.getDouble(inputColVector.asScratchTimestamp(0)), batchSize); } } else if (!batch.selectedInUse && inputColVector.noNulls) { @@ -378,7 +379,7 @@ private void iterateSelectionHasNulls( for (int j=0; j< batchSize; ++j) { int i = selected[j]; if (!isNull[i]) { - double value = inputColVector.getTimestampSecondsWithFractionalNanos(i); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(i)); if (myagg.isNull) { myagg.init (); } @@ -402,7 +403,7 @@ private void iterateSelectionNoNulls( myagg.init (); } - double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[0]); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selected[0])); myagg.sum += value; myagg.count += 1; if(myagg.count > 1) { @@ -413,7 +414,7 @@ private void iterateSelectionNoNulls( // i=0 was pulled out to remove the count > 1 check in the loop // for (int i=1; i< batchSize; ++i) { - value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]); + value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selected[i])); myagg.sum += value; myagg.count += 1; double t = myagg.count*value - myagg.sum; @@ -429,7 +430,7 @@ private void iterateNoSelectionHasNulls( for(int i=0;i 1 check for (int i=1; i 1) { @@ -322,7 +323,7 @@ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) if (inputColVector.isRepeating) { if (inputColVector.noNulls) { - iterateRepeatingNoNulls(myagg, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize); + iterateRepeatingNoNulls(myagg, TimestampWritable.getDouble(inputColVector.asScratchTimestamp(0)), batchSize); } } else if (!batch.selectedInUse && inputColVector.noNulls) { @@ -377,7 +378,7 @@ private void iterateSelectionHasNulls( for (int j=0; j< batchSize; ++j) { int i = selected[j]; if (!isNull[i]) { - double value = inputColVector.getTimestampSecondsWithFractionalNanos(i); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(i)); if (myagg.isNull) { myagg.init (); } @@ -401,7 +402,7 @@ private void iterateSelectionNoNulls( myagg.init (); } - double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[0]); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selected[0])); myagg.sum += value; myagg.count += 1; if(myagg.count > 1) { @@ -412,7 +413,7 @@ private void iterateSelectionNoNulls( // i=0 was pulled out to remove the count > 1 check in the loop // for (int i=1; i< batchSize; ++i) { - value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]); + value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selected[i])); myagg.sum += value; myagg.count += 1; double t = myagg.count*value - myagg.sum; @@ -428,7 +429,7 @@ private void iterateNoSelectionHasNulls( for(int i=0;i 1 check for (int i=1; i 1) { @@ -322,7 +323,7 @@ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) if (inputColVector.isRepeating) { if (inputColVector.noNulls) { - iterateRepeatingNoNulls(myagg, inputColVector.getTimestampSecondsWithFractionalNanos(0), batchSize); + iterateRepeatingNoNulls(myagg, TimestampWritable.getDouble(inputColVector.asScratchTimestamp(0)), batchSize); } } else if (!batch.selectedInUse && inputColVector.noNulls) { @@ -377,7 +378,7 @@ private void iterateSelectionHasNulls( for (int j=0; j< batchSize; ++j) { int i = selected[j]; if (!isNull[i]) { - double value = inputColVector.getTimestampSecondsWithFractionalNanos(i); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(i)); if (myagg.isNull) { myagg.init (); } @@ -401,7 +402,7 @@ private void iterateSelectionNoNulls( myagg.init (); } - double value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[0]); + double value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selected[0])); myagg.sum += value; myagg.count += 1; if(myagg.count > 1) { @@ -412,7 +413,7 @@ private void iterateSelectionNoNulls( // i=0 was pulled out to remove the count > 1 check in the loop // for (int i=1; i< batchSize; ++i) { - value = inputColVector.getTimestampSecondsWithFractionalNanos(selected[i]); + value = TimestampWritable.getDouble(inputColVector.asScratchTimestamp(selected[i])); myagg.sum += value; myagg.count += 1; double t = myagg.count*value - myagg.sum; @@ -428,7 +429,7 @@ private void iterateNoSelectionHasNulls( for(int i=0;i 1 check for (int i=1; i { + + // days/hours/minutes/seconds all represented as seconds + protected long totalSeconds; + protected int nanos; + + public HiveIntervalDayTime() { + } + + public HiveIntervalDayTime(int days, int hours, int minutes, int seconds, int nanos) { + set(days, hours, minutes, seconds, nanos); + } + + public HiveIntervalDayTime(long seconds, int nanos) { + set(seconds, nanos); + } + + public HiveIntervalDayTime(BigDecimal seconds) { + set(seconds); + } + + public HiveIntervalDayTime(HiveIntervalDayTime other) { + set(other.totalSeconds, other.nanos); + } + + public int getDays() { + return (int) TimeUnit.SECONDS.toDays(totalSeconds); + } + + public int getHours() { + return (int) (TimeUnit.SECONDS.toHours(totalSeconds) % TimeUnit.DAYS.toHours(1)); + } + + public int getMinutes() { + return (int) (TimeUnit.SECONDS.toMinutes(totalSeconds) % TimeUnit.HOURS.toMinutes(1)); + } + + public int getSeconds() { + return (int) (totalSeconds % TimeUnit.MINUTES.toSeconds(1)); + } + + public int getNanos() { + return nanos; + } + + /** + * Returns days/hours/minutes all converted into seconds. + * Nanos still need to be retrieved using getNanos() + * @return + */ + public long getTotalSeconds() { + return totalSeconds; + } + + /** + * Ensures that the seconds and nanoseconds fields have consistent sign + */ + protected void normalizeSecondsAndNanos() { + if (totalSeconds > 0 && nanos < 0) { + --totalSeconds; + nanos += IntervalDayTimeUtils.NANOS_PER_SEC; + } else if (totalSeconds < 0 && nanos > 0) { + ++totalSeconds; + nanos -= IntervalDayTimeUtils.NANOS_PER_SEC; + } + } + + public void set(int days, int hours, int minutes, int seconds, int nanos) { + long totalSeconds = seconds; + totalSeconds += TimeUnit.DAYS.toSeconds(days); + totalSeconds += TimeUnit.HOURS.toSeconds(hours); + totalSeconds += TimeUnit.MINUTES.toSeconds(minutes); + totalSeconds += TimeUnit.NANOSECONDS.toSeconds(nanos); + nanos = nanos % IntervalDayTimeUtils.NANOS_PER_SEC; + + this.totalSeconds = totalSeconds; + this.nanos = nanos; + + normalizeSecondsAndNanos(); + } + + public void set(long seconds, int nanos) { + this.totalSeconds = seconds; + this.nanos = nanos; + normalizeSecondsAndNanos(); + } + + public void set(BigDecimal totalSecondsBd) { + long totalSeconds = totalSecondsBd.longValue(); + BigDecimal fractionalSecs = totalSecondsBd.remainder(BigDecimal.ONE); + int nanos = fractionalSecs.multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue(); + set(totalSeconds, nanos); + } + + public void set(HiveIntervalDayTime other) { + set(other.getTotalSeconds(), other.getNanos()); + } + + public HiveIntervalDayTime negate() { + return new HiveIntervalDayTime(-getTotalSeconds(), -getNanos()); + } + + @Override + public int compareTo(HiveIntervalDayTime other) { + long cmp = this.totalSeconds - other.totalSeconds; + if (cmp == 0) { + cmp = this.nanos - other.nanos; + } + if (cmp != 0) { + cmp = cmp > 0 ? 1 : -1; + } + return (int) cmp; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof HiveIntervalDayTime)) { + return false; + } + return 0 == compareTo((HiveIntervalDayTime) obj); + } + + /** + * Return a copy of this object. + */ + public Object clone() { + return new HiveIntervalDayTime(totalSeconds, nanos); + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(totalSeconds).append(nanos).toHashCode(); + } + + @Override + public String toString() { + // If normalize() was used, then day-hour-minute-second-nanos should have the same sign. + // This is currently working with that assumption. + boolean isNegative = (totalSeconds < 0 || nanos < 0); + String daySecondSignStr = isNegative ? "-" : ""; + + return String.format("%s%d %02d:%02d:%02d.%09d", + daySecondSignStr, Math.abs(getDays()), + Math.abs(getHours()), Math.abs(getMinutes()), + Math.abs(getSeconds()), Math.abs(getNanos())); + } + + public static HiveIntervalDayTime valueOf(String strVal) { + HiveIntervalDayTime result = null; + if (strVal == null) { + throw new IllegalArgumentException("Interval day-time string was null"); + } + Matcher patternMatcher = PATTERN_MATCHER.get(); + patternMatcher.reset(strVal); + if (patternMatcher.matches()) { + // Parse out the individual parts + try { + // Sign - whether interval is positive or negative + int sign = 1; + String field = patternMatcher.group(1); + if (field != null && field.equals("-")) { + sign = -1; + } + int days = sign * + IntervalDayTimeUtils.parseNumericValueWithRange("day", patternMatcher.group(2), + 0, Integer.MAX_VALUE); + byte hours = (byte) (sign * + IntervalDayTimeUtils.parseNumericValueWithRange("hour", patternMatcher.group(3), 0, 23)); + byte minutes = (byte) (sign * + IntervalDayTimeUtils.parseNumericValueWithRange("minute", patternMatcher.group(4), 0, 59)); + int seconds = 0; + int nanos = 0; + field = patternMatcher.group(5); + if (field != null) { + BigDecimal bdSeconds = new BigDecimal(field); + if (bdSeconds.compareTo(IntervalDayTimeUtils.MAX_INT_BD) > 0) { + throw new IllegalArgumentException("seconds value of " + bdSeconds + " too large"); + } + seconds = sign * bdSeconds.intValue(); + nanos = sign * bdSeconds.subtract(new BigDecimal(bdSeconds.toBigInteger())) + .multiply(IntervalDayTimeUtils.NANOS_PER_SEC_BD).intValue(); + } + + result = new HiveIntervalDayTime(days, hours, minutes, seconds, nanos); + } catch (Exception err) { + throw new IllegalArgumentException("Error parsing interval day-time string: " + strVal, err); + } + } else { + throw new IllegalArgumentException( + "Interval string does not match day-time format of 'd h:m:s.n': " + strVal); + } + + return result; + } + + // Simple pattern: D H:M:S.nnnnnnnnn + private final static String PARSE_PATTERN = + "([+|-])?(\\d+) (\\d+):(\\d+):((\\d+)(\\.(\\d+))?)"; + + private static final ThreadLocal PATTERN_MATCHER = new ThreadLocal() { + @Override + protected Matcher initialValue() { + return Pattern.compile(PARSE_PATTERN).matcher(""); + } + }; +} diff --git storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java deleted file mode 100644 index ac1e38a..0000000 --- storage-api/src/java/org/apache/hadoop/hive/common/type/PisaTimestamp.java +++ /dev/null @@ -1,609 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.common.type; - -import java.math.BigDecimal; -import java.sql.Timestamp; -import java.util.Random; -import java.util.concurrent.TimeUnit; - -import com.google.common.base.Preconditions; - -/** - * Pisa project is named after the famous Leonardo of Pisa, or better known as Fibanacci. - * - * A Pisa timestamp is a timestamp without a time-zone (i.e. local) in the ISO-8601 calendar system, - * such as 2007-12-03 10:15:30.0123456789, with accuracy to the nanosecond (1 billionth of a - * second). - * - * Pisa timestamps use the same starting point as a java.sql.Timestamp -- the number of nanoseconds - * since the epoch (1970-01-01, or the day Unix roared awake) where negative numbers represent - * earlier days. - * - * However, we use the PisaTimestamp class which has different design requirements than - * java.sql.Timestamp. It is designed to be mutable and NOT thread-safe to avoid high memory - * allocation / garbage collection costs. And, provides for ease of use by our vectorization - * code to avoid the high CPU data cache miss cost for small objects, too. We do this by allowing - * the epoch day and nano of day to be stored externally (i.e. vector arrays). - * - * And, importantly, PisaTimestamp is a light-weight class similar to the epochDay/NanoOfDay of - * the newer Java 8 LocalDateTime class, except the timestamp is *indifferent* to timezone. - * - * A common usage would be to treat it as UTC. - * - * You can work with days, seconds, milliseconds, nanoseconds, etc. But to work with months you - * will need to convert to an external timestamp object and use calendars, etc. - * * - * The storage for a PisaTimestamp is: - * - * long epochDay - * // The number of days since 1970-01-01 (==> similar to Java 8 LocalDate). - * long nanoOfDay - * // The number of nanoseconds within the day, with the range of - * // 0 to 24 * 60 * 60 * 1,000,000,000 - 1 (==> similar to Java 8 LocalTime). - * - * Both epochDay and nanoOfDay are signed. - * - * We when both epochDay and nanoOfDay are non-zero, we will maintain them so they have the - * same sign. - * - */ - -public class PisaTimestamp { - - private static final long serialVersionUID = 1L; - - private long epochDay; - private long nanoOfDay; - - private Timestamp scratchTimestamp; - - public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1); - public static final long NANOSECONDS_PER_MILLISECOND = TimeUnit.MILLISECONDS.toNanos(1); - public static final long NANOSECONDS_PER_DAY = TimeUnit.DAYS.toNanos(1); - - public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1); - public static final long MILLISECONDS_PER_DAY = TimeUnit.DAYS.toMillis(1); - - public static final long SECONDS_PER_DAY = TimeUnit.DAYS.toSeconds(1); - - public static final long MIN_NANO_OF_DAY = -NANOSECONDS_PER_DAY; - public static final long MAX_NANO_OF_DAY = NANOSECONDS_PER_DAY; - - public static final BigDecimal BIG_NANOSECONDS_PER_SECOND = new BigDecimal(NANOSECONDS_PER_SECOND); - - public long getEpochDay() { - return epochDay; - } - - public long getNanoOfDay() { - return nanoOfDay; - } - - public PisaTimestamp() { - epochDay = 0; - nanoOfDay = 0; - scratchTimestamp = new Timestamp(0); - } - - public PisaTimestamp(long epochDay, long nanoOfDay) { - - Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay), - "epochDay " + epochDay + ", nanoOfDay " + nanoOfDay + " not valid"); - - this.epochDay = epochDay; - this.nanoOfDay = nanoOfDay; - scratchTimestamp = new Timestamp(0); - } - - public PisaTimestamp(Timestamp timestamp) { - super(); - updateFromTimestamp(timestamp); - } - - public void reset() { - epochDay = 0; - nanoOfDay = 0; - } - - /** - * NOTE: This method validates the integrity rules between epoch day and nano of day, - * but not overflow/underflow of epoch day. Since epoch day overflow/underflow can result - * from to client data input, that must be checked manually with as this - * class do not throw data range exceptions as a rule. It leaves that choice to the caller. - * @param epochDay - * @param nanoOfDay - * @return true if epoch day and nano of day have integrity. - */ - public static boolean validateIntegrity(long epochDay, long nanoOfDay) { - - // Range check nano per day as invariant. - if (nanoOfDay >= NANOSECONDS_PER_DAY || nanoOfDay <= -NANOSECONDS_PER_DAY) { - return false; - } - - // Signs of epoch day and nano of day must match. - if (!(epochDay >= 0 && nanoOfDay >= 0 || - epochDay <= 0 && nanoOfDay <= 0)) { - return false; - } - - return true; - } - - /** - * Set this PisaTimestamp from another PisaTimestamp. - * @param source - * @return this - */ - public PisaTimestamp update(PisaTimestamp source) { - this.epochDay = source.epochDay; - this.nanoOfDay = source.nanoOfDay; - return this; - } - - /** - * Set this PisaTimestamp from a epoch day and nano of day. - * @param epochDay - * @param nanoOfDay - * @return this - */ - public PisaTimestamp update(long epochDay, long nanoOfDay) { - - Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay), - "epochDay " + epochDay + ", nanoOfDay " + nanoOfDay + " not valid"); - - this.epochDay = epochDay; - this.nanoOfDay = nanoOfDay; - return this; - } - - /** - * Set the PisaTimestamp from a Timestamp object. - * @param timestamp - * @return this - */ - public PisaTimestamp updateFromTimestamp(Timestamp timestamp) { - - long timestampTime = timestamp.getTime(); - int nanos = timestamp.getNanos(); - - /** - * Since the Timestamp class always stores nanos as a positive quantity (0 .. 999,999,999), - * we have to adjust back the time (subtract) by 1,000,000,000 to get right quantity for - * our calculations below. One thing it ensures is nanoOfDay will be negative. - */ - if (timestampTime < 0 && nanos > 0) { - timestampTime -= MILLISECONDS_PER_SECOND; - } - - // The Timestamp class does not use the milliseconds part (always 0). It is covered by nanos. - long epochSeconds = timestampTime / MILLISECONDS_PER_SECOND; - - nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND + nanos; - epochDay = epochSeconds / SECONDS_PER_DAY + (nanoOfDay / NANOSECONDS_PER_DAY); - - Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay)); - return this; - } - - /** - * Set this PisaTimestamp from a timestamp milliseconds. - * @param epochMilliseconds - * @return this - */ - public PisaTimestamp updateFromTimestampMilliseconds(long timestampMilliseconds) { - /** - * The Timestamp class setTime sets both the time (seconds stored as milliseconds) and - * the nanos. - */ - scratchTimestamp.setTime(timestampMilliseconds); - updateFromTimestamp(scratchTimestamp); - return this; - } - - /** - * Set this PisaTimestamp from a timestamp seconds. - * @param epochMilliseconds - * @return this - */ - public PisaTimestamp updateFromTimestampSeconds(long timestampSeconds) { - return updateFromTimestampMilliseconds(timestampSeconds * MILLISECONDS_PER_SECOND); - } - - /** - * Set this PisaTimestamp from a timestamp seconds. - * @param epochMilliseconds - * @return this - */ - public PisaTimestamp updateFromTimestampSecondsWithFractionalNanoseconds( - double timestampSecondsWithFractionalNanoseconds) { - - // Otherwise, BigDecimal throws an exception. (Support vector operations that sometimes - // do work on double Not-a-Number NaN values). - if (Double.isNaN(timestampSecondsWithFractionalNanoseconds)) { - timestampSecondsWithFractionalNanoseconds = 0; - } - // Algorithm used by TimestampWritable.doubleToTimestamp method. - // Allocates a BigDecimal object! - - long seconds = (long) timestampSecondsWithFractionalNanoseconds; - - // We must ensure the exactness of the double's fractional portion. - // 0.6 as the fraction part will be converted to 0.59999... and - // significantly reduce the savings from binary serialization. - BigDecimal bd; - - bd = new BigDecimal(String.valueOf(timestampSecondsWithFractionalNanoseconds)); - bd = bd.subtract(new BigDecimal(seconds)); // Get the nanos fraction. - bd = bd.multiply(BIG_NANOSECONDS_PER_SECOND); // Make nanos an integer. - - int nanos = bd.intValue(); - - // Convert to millis - long millis = seconds * 1000; - if (nanos < 0) { - millis -= 1000; - nanos += 1000000000; - } - - scratchTimestamp.setTime(millis); - scratchTimestamp.setNanos(nanos); - updateFromTimestamp(scratchTimestamp); - return this; - } - - /** - * Set this PisaTimestamp from a epoch seconds and signed nanos (-999999999 to 999999999). - * @param epochSeconds - * @param signedNanos - * @return this - */ - public PisaTimestamp updateFromEpochSecondsAndSignedNanos(long epochSeconds, int signedNanos) { - - long nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND + signedNanos; - long epochDay = epochSeconds / SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_DAY; - - Preconditions.checkState(validateIntegrity(epochDay, nanoOfDay)); - - this.epochDay = epochDay; - this.nanoOfDay = nanoOfDay; - return this; - } - - /** - * Set a scratch PisaTimestamp with this PisaTimestamp's values and return the scratch object. - * @param epochDay - * @param nanoOfDay - */ - public PisaTimestamp scratchCopy(PisaTimestamp scratch) { - - scratch.epochDay = epochDay; - scratch.nanoOfDay = nanoOfDay; - return scratch; - } - - /** - * Set a Timestamp object from this PisaTimestamp. - * @param timestamp - */ - public void timestampUpdate(Timestamp timestamp) { - - /* - * java.sql.Timestamp consists of a long variable to store milliseconds and an integer variable for nanoseconds. - * The long variable is used to store only the full seconds converted to millis. For example for 1234 milliseconds, - * 1000 is stored in the long variable, and 234000000 (234 converted to nanoseconds) is stored as nanoseconds. - * The negative timestamps are also supported, but nanoseconds must be positive therefore millisecond part is - * reduced by one second. - */ - - long epochSeconds = epochDay * SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_SECOND; - long integralSecInMillis; - int nanos = (int) (nanoOfDay % NANOSECONDS_PER_SECOND); // The nanoseconds. - if (nanos < 0) { - nanos = (int) NANOSECONDS_PER_SECOND + nanos; // The positive nano-part that will be added to milliseconds. - integralSecInMillis = (epochSeconds - 1) * MILLISECONDS_PER_SECOND; // Reduce by one second. - } else { - integralSecInMillis = epochSeconds * MILLISECONDS_PER_SECOND; // Full seconds converted to millis. - } - - timestamp.setTime(integralSecInMillis); - timestamp.setNanos(nanos); - } - - /** - * Return the scratch timestamp with values from Pisa timestamp. - * @return - */ - public Timestamp asScratchTimestamp() { - timestampUpdate(scratchTimestamp); - return scratchTimestamp; - } - - /** - * Return the scratch timestamp for use by the caller. - * @return - */ - public Timestamp useScratchTimestamp() { - return scratchTimestamp; - } - - public int compareTo(PisaTimestamp another) { - - if (epochDay == another.epochDay) { - if (nanoOfDay == another.nanoOfDay){ - return 0; - } else { - return (nanoOfDay < another.nanoOfDay ? -1 : 1); - } - } else { - return (epochDay < another.epochDay ? -1: 1); - } - } - - public static int compareTo(long epochDay1, long nanoOfDay1, PisaTimestamp another) { - - if (epochDay1 == another.epochDay) { - if (nanoOfDay1 == another.nanoOfDay){ - return 0; - } else { - return (nanoOfDay1 < another.nanoOfDay ? -1 : 1); - } - } else { - return (epochDay1 < another.epochDay ? -1: 1); - } - } - - public static int compareTo(PisaTimestamp pisaTimestamp1, long epochDay2, long nanoOfDay2) { - - if (pisaTimestamp1.epochDay == epochDay2) { - if (pisaTimestamp1.nanoOfDay == nanoOfDay2){ - return 0; - } else { - return (pisaTimestamp1.nanoOfDay < nanoOfDay2 ? -1 : 1); - } - } else { - return (pisaTimestamp1.epochDay < epochDay2 ? -1: 1); - } - } - - public static int compareTo(long epochDay1, long nanoOfDay1, long epochDay2, long nanoOfDay2) { - - if (epochDay1 == epochDay2) { - if (nanoOfDay1 == nanoOfDay2){ - return 0; - } else { - return (nanoOfDay1 < nanoOfDay2 ? -1 : 1); - } - } else { - return (epochDay1 < epochDay2 ? -1: 1); - } - } - - - /** - * Standard equals method override. - */ - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - return equals((PisaTimestamp) obj); - } - - public boolean equals(PisaTimestamp other) { - - if (epochDay == other.epochDay) { - if (nanoOfDay == other.nanoOfDay) { - return true; - } else { - return false; - } - } else { - return false; - } - } - - public static void add(PisaTimestamp pisaTimestamp1, PisaTimestamp pisaTimestamp2, - PisaTimestamp result) { - add(pisaTimestamp1.epochDay, pisaTimestamp1.nanoOfDay, - pisaTimestamp2.epochDay, pisaTimestamp2.nanoOfDay, - result); - } - - public static void add(long epochDay1, long nanoOfDay1, - long epochDay2, long nanoOfDay2, - PisaTimestamp result) { - - // Validate integrity rules between epoch day and nano of day. - Preconditions.checkState(PisaTimestamp.validateIntegrity(epochDay1, nanoOfDay1)); - Preconditions.checkState(PisaTimestamp.validateIntegrity(epochDay2, nanoOfDay2)); - - long intermediateEpochDay = epochDay1 + epochDay2; - long intermediateNanoOfDay = nanoOfDay1 + nanoOfDay2; - - // Normalize so both are positive or both are negative. - long normalizedEpochDay; - long normalizedNanoOfDay; - if (intermediateEpochDay > 0 && intermediateNanoOfDay < 0) { - normalizedEpochDay = intermediateEpochDay - 1; - normalizedNanoOfDay = intermediateNanoOfDay + NANOSECONDS_PER_DAY; - } else if (intermediateEpochDay < 0 && intermediateNanoOfDay > 0) { - normalizedEpochDay = intermediateEpochDay + 1; - normalizedNanoOfDay = intermediateNanoOfDay - NANOSECONDS_PER_DAY; - } else { - normalizedEpochDay = intermediateEpochDay; - normalizedNanoOfDay = intermediateNanoOfDay; - } - - long resultEpochDay; - long resultNanoOfDay; - if (normalizedNanoOfDay >= NANOSECONDS_PER_DAY || normalizedNanoOfDay <= -NANOSECONDS_PER_DAY) { - // Adjust for carry or overflow... - - resultEpochDay = normalizedEpochDay + normalizedNanoOfDay / NANOSECONDS_PER_DAY; - resultNanoOfDay = normalizedNanoOfDay % NANOSECONDS_PER_DAY; - - } else { - resultEpochDay = normalizedEpochDay; - resultNanoOfDay = normalizedNanoOfDay; - } - - // The update method will validate integrity rules between epoch day and nano of day, - // but not overflow/underflow of epoch day. - result.update(resultEpochDay, resultNanoOfDay); - } - - public static void addSeconds(PisaTimestamp timestamp1, long epochSeconds, PisaTimestamp result) { - long epochDay = epochSeconds / SECONDS_PER_DAY; - long nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND; - add(timestamp1.epochDay, timestamp1.nanoOfDay, epochDay, nanoOfDay, result); - } - - public static void subtract(PisaTimestamp timestamp1, PisaTimestamp timestamp2, - PisaTimestamp result) { - - add(timestamp1.epochDay, timestamp1.nanoOfDay, -timestamp2.epochDay, -timestamp2.nanoOfDay, - result); - } - - public static void subtract(long epochDay1, long nanoOfDay1, - long epochDay2, long nanoOfDay2, - PisaTimestamp result) { - - add(epochDay1, nanoOfDay1, -epochDay2, -nanoOfDay2, result); - } - - public static void subtractSeconds(PisaTimestamp timestamp1, long epochSeconds, - PisaTimestamp result) { - long epochDay = epochSeconds / SECONDS_PER_DAY; - long nanoOfDay = (epochSeconds % SECONDS_PER_DAY) * NANOSECONDS_PER_SECOND; - add(timestamp1.epochDay, timestamp1.nanoOfDay, -epochDay, -nanoOfDay, result); - } - - /** - * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of - * seconds. 500 would round to 0, -500 would round to -1. - */ - public static long timestampMillisToSeconds(long millis) { - if (millis >= 0) { - return millis / 1000; - } else { - return (millis - 999) / 1000; - } - } - - /** - * Return a double with the integer part as the seconds and the fractional part as - * the nanoseconds the way the Timestamp class does it. - * @return seconds.nanoseconds - */ - public double getTimestampSecondsWithFractionalNanos() { - // Algorithm must be the same as TimestampWritable.getDouble method. - timestampUpdate(scratchTimestamp); - double seconds = timestampMillisToSeconds(scratchTimestamp.getTime()); - double nanos = scratchTimestamp.getNanos(); - BigDecimal bigSeconds = new BigDecimal(seconds); - BigDecimal bigNanos = new BigDecimal(nanos).divide(BIG_NANOSECONDS_PER_SECOND); - return bigSeconds.add(bigNanos).doubleValue(); - } - - /** - * Return an integer as the seconds the way the Timestamp class does it. - * @return seconds.nanoseconds - */ - public long getTimestampSeconds() { - // Algorithm must be the same as TimestampWritable.getSeconds method. - timestampUpdate(scratchTimestamp); - return timestampMillisToSeconds(scratchTimestamp.getTime()); - } - - /** - * Return an integer as the milliseconds the way the Timestamp class does it. - * @return seconds.nanoseconds - */ - public long getTimestampMilliseconds() { - timestampUpdate(scratchTimestamp); - return scratchTimestamp.getTime(); - } - - /** - * Return the epoch seconds. - * @return - */ - public long getEpochSeconds() { - return epochDay * SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_SECOND; - } - - /** - * Return the epoch seconds, given the epoch day and nano of day. - * @param epochDay - * @param nanoOfDay - * @return - */ - public static long getEpochSecondsFromEpochDayAndNanoOfDay(long epochDay, long nanoOfDay) { - return epochDay * SECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_SECOND; - } - - /** - * Return the signed nanos (-999999999 to 999999999). - * NOTE: Not the same as Timestamp class nanos (which are always positive). - */ - public int getSignedNanos() { - return (int) (nanoOfDay % NANOSECONDS_PER_SECOND); - } - - /** - * Return the signed nanos (-999999999 to 999999999). - * NOTE: Not the same as Timestamp class nanos (which are always positive). - */ - public static int getSignedNanos(long nanoOfDay) { - return (int) (nanoOfDay % NANOSECONDS_PER_SECOND); - } - - /** - * Return the epoch milliseconds. - * @return - */ - public long getEpochMilliseconds() { - return epochDay * MILLISECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_MILLISECOND; - } - - /** - * Return the epoch seconds, given the epoch day and nano of day. - * @param epochDay - * @param nanoOfDay - * @return - */ - public static long getEpochMillisecondsFromEpochDayAndNanoOfDay(long epochDay, long nanoOfDay) { - return epochDay * MILLISECONDS_PER_DAY + nanoOfDay / NANOSECONDS_PER_MILLISECOND; - } - - @Override - public int hashCode() { - // UNDONE: We don't want to box the longs just to get the hash codes... - return new Long(epochDay).hashCode() ^ new Long(nanoOfDay).hashCode(); - } - - @Override - public String toString() { - timestampUpdate(scratchTimestamp); - return scratchTimestamp.toString(); - } -} \ No newline at end of file diff --git storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java index 13baff4..1c1d856 100644 --- storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java +++ storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java @@ -18,21 +18,64 @@ package org.apache.hadoop.hive.common.type; import java.sql.Timestamp; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Random; import java.util.concurrent.TimeUnit; public class RandomTypeUtil { public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1); + public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1); + public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1); + + private static ThreadLocal DATE_FORMAT = + new ThreadLocal() { + @Override + protected DateFormat initialValue() { + return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + } + }; + + private static long MIN_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("0001-01-01 00:00:00"); + private static long MAX_FOUR_DIGIT_YEAR_MILLIS = parseToMillis("9999-01-01 00:00:00"); + + private static long parseToMillis(String s) { + try { + return DATE_FORMAT.get().parse(s).getTime(); + } catch (ParseException ex) { + throw new RuntimeException(ex); + } + } public static Timestamp getRandTimestamp(Random r) { + return getRandTimestamp(r, 0000, 9999); + } + + public static Timestamp getRandTimestamp(Random r, int minYear, int maxYear) { String optionalNanos = ""; - if (r.nextInt(2) == 1) { + switch (r.nextInt(4)) { + case 0: + // No nanos. + break; + case 1: + optionalNanos = String.format(".%09d", + Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_SECOND))); + break; + case 2: + // Limit to milliseconds only... optionalNanos = String.format(".%09d", - Integer.valueOf(0 + r.nextInt((int) NANOSECONDS_PER_SECOND))); + Integer.valueOf(r.nextInt((int) MILLISECONDS_PER_SECOND)) * NANOSECONDS_PER_MILLISSECOND); + break; + case 3: + // Limit to below milliseconds only... + optionalNanos = String.format(".%09d", + Integer.valueOf(r.nextInt((int) NANOSECONDS_PER_MILLISSECOND))); + break; } String timestampStr = String.format("%04d-%02d-%02d %02d:%02d:%02d%s", - Integer.valueOf(0 + r.nextInt(10000)), // year + Integer.valueOf(minYear + r.nextInt(maxYear - minYear + 1)), // year Integer.valueOf(1 + r.nextInt(12)), // month Integer.valueOf(1 + r.nextInt(28)), // day Integer.valueOf(0 + r.nextInt(24)), // hour @@ -48,4 +91,22 @@ public static Timestamp getRandTimestamp(Random r) { } return timestampVal; } + + public static long randomMillis(long minMillis, long maxMillis, Random rand) { + return minMillis + (long) ((maxMillis - minMillis) * rand.nextDouble()); + } + + public static long randomMillis(Random rand) { + return randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS, rand); + } + + public static int randomNanos(Random rand, int decimalDigits) { + // Only keep the most significant decimalDigits digits. + int nanos = rand.nextInt((int) NANOSECONDS_PER_SECOND); + return nanos - nanos % (int) Math.pow(10, 9 - decimalDigits); + } + + public static int randomNanos(Random rand) { + return randomNanos(rand, 9); + } } \ No newline at end of file diff --git storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java index 4ae9c47..c069a5f 100644 --- storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java +++ storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.exec.vector; -import java.io.IOException; import java.util.Arrays; /** @@ -43,6 +42,7 @@ BYTES, DECIMAL, TIMESTAMP, + INTERVAL_DAY_TIME, STRUCT, LIST, MAP, diff --git storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java new file mode 100644 index 0000000..69cf130 --- /dev/null +++ storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java @@ -0,0 +1,338 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector; + +import java.util.Arrays; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; +import org.apache.hadoop.io.Writable; + +/** + * This class represents a nullable interval day time column vector capable of handing a + * wide range of interval day time values. + * + * We store the 2 (value) fields of a HiveIntervalDayTime class in primitive arrays. + * + * We do this to avoid an array of Java HiveIntervalDayTime objects which would have poor storage + * and memory access characteristics. + * + * Generally, the caller will fill in a scratch HiveIntervalDayTime object with values from a row, + * work using the scratch HiveIntervalDayTime, and then perhaps update the column vector row + * with a result. + */ +public class IntervalDayTimeColumnVector extends ColumnVector { + + /* + * The storage arrays for this column vector corresponds to the storage of a HiveIntervalDayTime: + */ + private long[] totalSeconds; + // The values from HiveIntervalDayTime.getTotalSeconds(). + + private int[] nanos; + // The values from HiveIntervalDayTime.getNanos(). + + /* + * Scratch objects. + */ + private final HiveIntervalDayTime scratchIntervalDayTime; + + private Writable scratchWritable; + // Supports keeping a HiveIntervalDayTimeWritable object without having to import + // that definition... + + /** + * Use this constructor by default. All column vectors + * should normally be the default size. + */ + public IntervalDayTimeColumnVector() { + this(VectorizedRowBatch.DEFAULT_SIZE); + } + + /** + * Don't use this except for testing purposes. + * + * @param len the number of rows + */ + public IntervalDayTimeColumnVector(int len) { + super(len); + + totalSeconds = new long[len]; + nanos = new int[len]; + + scratchIntervalDayTime = new HiveIntervalDayTime(); + + scratchWritable = null; // Allocated by caller. + } + + /** + * Return the number of rows. + * @return + */ + public int getLength() { + return totalSeconds.length; + } + + /** + * Return a row's HiveIntervalDayTime.getTotalSeconds() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public long getTotalSeconds(int elementNum) { + return totalSeconds[elementNum]; + } + + /** + * Return a row's HiveIntervalDayTime.getNanos() value. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public long getNanos(int elementNum) { + return nanos[elementNum]; + } + + /** + * Set a HiveIntervalDayTime object from a row of the column. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param intervalDayTime + * @param elementNum + */ + public void timestampUpdate(HiveIntervalDayTime intervalDayTime, int elementNum) { + intervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]); + } + + + /** + * Return the scratch HiveIntervalDayTime object set from a row. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @return + */ + public HiveIntervalDayTime asScratchIntervalDayTime(int elementNum) { + scratchIntervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]); + return scratchIntervalDayTime; + } + + /** + * Return the scratch HiveIntervalDayTime (contents undefined). + * @return + */ + public HiveIntervalDayTime getScratchIntervalDayTime() { + return scratchIntervalDayTime; + } + + /** + * Compare row to HiveIntervalDayTime. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param elementNum + * @param intervalDayTime + * @return -1, 0, 1 standard compareTo values. + */ + public int compareTo(int elementNum, HiveIntervalDayTime intervalDayTime) { + return asScratchIntervalDayTime(elementNum).compareTo(intervalDayTime); + } + + /** + * Compare HiveIntervalDayTime to row. + * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param intervalDayTime + * @param elementNum + * @return -1, 0, 1 standard compareTo values. + */ + public int compareTo(HiveIntervalDayTime intervalDayTime, int elementNum) { + return intervalDayTime.compareTo(asScratchIntervalDayTime(elementNum)); + } + + /** + * Compare a row to another TimestampColumnVector's row. + * @param elementNum1 + * @param intervalDayTimeColVector2 + * @param elementNum2 + * @return + */ + public int compareTo(int elementNum1, IntervalDayTimeColumnVector intervalDayTimeColVector2, + int elementNum2) { + return asScratchIntervalDayTime(elementNum1).compareTo( + intervalDayTimeColVector2.asScratchIntervalDayTime(elementNum2)); + } + + /** + * Compare another TimestampColumnVector's row to a row. + * @param intervalDayTimeColVector1 + * @param elementNum1 + * @param elementNum2 + * @return + */ + public int compareTo(IntervalDayTimeColumnVector intervalDayTimeColVector1, int elementNum1, + int elementNum2) { + return intervalDayTimeColVector1.asScratchIntervalDayTime(elementNum1).compareTo( + asScratchIntervalDayTime(elementNum2)); + } + + @Override + public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) { + + IntervalDayTimeColumnVector timestampColVector = (IntervalDayTimeColumnVector) inputVector; + + totalSeconds[outElementNum] = timestampColVector.totalSeconds[inputElementNum]; + nanos[outElementNum] = timestampColVector.nanos[inputElementNum]; + } + + // Simplify vector by brute-force flattening noNulls and isRepeating + // This can be used to reduce combinatorial explosion of code paths in VectorExpressions + // with many arguments. + public void flatten(boolean selectedInUse, int[] sel, int size) { + flattenPush(); + if (isRepeating) { + isRepeating = false; + long repeatFastTime = totalSeconds[0]; + int repeatNanos = nanos[0]; + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + totalSeconds[i] = repeatFastTime; + nanos[i] = repeatNanos; + } + } else { + Arrays.fill(totalSeconds, 0, size, repeatFastTime); + Arrays.fill(nanos, 0, size, repeatNanos); + } + flattenRepeatingNulls(selectedInUse, sel, size); + } + flattenNoNulls(selectedInUse, sel, size); + } + + /** + * Set a row from a HiveIntervalDayTime. + * We assume the entry has already been isRepeated adjusted. + * @param elementNum + * @param intervalDayTime + */ + public void set(int elementNum, HiveIntervalDayTime intervalDayTime) { + this.totalSeconds[elementNum] = intervalDayTime.getTotalSeconds(); + this.nanos[elementNum] = intervalDayTime.getNanos(); + } + + /** + * Set a row from the current value in the scratch interval day time. + * @param elementNum + */ + public void setFromScratchIntervalDayTime(int elementNum) { + this.totalSeconds[elementNum] = scratchIntervalDayTime.getTotalSeconds(); + this.nanos[elementNum] = scratchIntervalDayTime.getNanos(); + } + + /** + * Set row to standard null value(s). + * We assume the entry has already been isRepeated adjusted. + * @param elementNum + */ + public void setNullValue(int elementNum) { + totalSeconds[elementNum] = 0; + nanos[elementNum] = 1; + } + + // Copy the current object contents into the output. Only copy selected entries, + // as indicated by selectedInUse and the sel array. + public void copySelected( + boolean selectedInUse, int[] sel, int size, IntervalDayTimeColumnVector output) { + + // Output has nulls if and only if input has nulls. + output.noNulls = noNulls; + output.isRepeating = false; + + // Handle repeating case + if (isRepeating) { + output.totalSeconds[0] = totalSeconds[0]; + output.nanos[0] = nanos[0]; + output.isNull[0] = isNull[0]; + output.isRepeating = true; + return; + } + + // Handle normal case + + // Copy data values over + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.totalSeconds[i] = totalSeconds[i]; + output.nanos[i] = nanos[i]; + } + } + else { + System.arraycopy(totalSeconds, 0, output.totalSeconds, 0, size); + System.arraycopy(nanos, 0, output.nanos, 0, size); + } + + // Copy nulls over if needed + if (!noNulls) { + if (selectedInUse) { + for (int j = 0; j < size; j++) { + int i = sel[j]; + output.isNull[i] = isNull[i]; + } + } + else { + System.arraycopy(isNull, 0, output.isNull, 0, size); + } + } + } + + /** + * Fill all the vector entries with a HiveIntervalDayTime. + * @param intervalDayTime + */ + public void fill(HiveIntervalDayTime intervalDayTime) { + noNulls = true; + isRepeating = true; + totalSeconds[0] = intervalDayTime.getTotalSeconds(); + nanos[0] = intervalDayTime.getNanos(); + } + + /** + * Return a convenience writable object stored by this column vector. + * Supports keeping a TimestampWritable object without having to import that definition... + * @return + */ + public Writable getScratchWritable() { + return scratchWritable; + } + + /** + * Set the convenience writable object stored by this column vector + * @param scratchWritable + */ + public void setScratchWritable(Writable scratchWritable) { + this.scratchWritable = scratchWritable; + } + + @Override + public void stringifyValue(StringBuilder buffer, int row) { + if (isRepeating) { + row = 0; + } + if (noNulls || !isNull[row]) { + scratchIntervalDayTime.set(totalSeconds[row], nanos[row]); + buffer.append(scratchIntervalDayTime.toString()); + } else { + buffer.append("null"); + } + } +} \ No newline at end of file diff --git storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java index b73a0d2..fcdaf95 100644 --- storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java +++ storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java @@ -20,37 +20,35 @@ import java.sql.Timestamp; import java.util.Arrays; -import org.apache.hadoop.hive.common.type.PisaTimestamp; -import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.io.Writable; -import com.google.common.base.Preconditions; - /** * This class represents a nullable timestamp column vector capable of handing a wide range of * timestamp values. * - * We use the PisaTimestamp which is designed to be mutable and avoid the heavy memory allocation - * and CPU data cache miss costs. + * We store the 2 (value) fields of a Timestamp class in primitive arrays. + * + * We do this to avoid an array of Java Timestamp objects which would have poor storage + * and memory access characteristics. + * + * Generally, the caller will fill in a scratch timestamp object with values from a row, work + * using the scratch timestamp, and then perhaps update the column vector row with a result. */ public class TimestampColumnVector extends ColumnVector { /* - * The storage arrays for this column vector corresponds to the storage of a PisaTimestamp: + * The storage arrays for this column vector corresponds to the storage of a Timestamp: */ - private long[] epochDay; - // An array of the number of days since 1970-01-01 (similar to Java 8 LocalDate). + private long[] time; + // The values from Timestamp.getValue(). - private long[] nanoOfDay; - // An array of the number of nanoseconds within the day, with the range of - // 0 to 24 * 60 * 60 * 1,000,000,000 - 1 (similar to Java 8 LocalTime). + private int[] nanos; + // The values from Timestamp.getNanos(). /* * Scratch objects. */ - private PisaTimestamp scratchPisaTimestamp; - // Convenience scratch Pisa timestamp object. + private final Timestamp scratchTimestamp; private Writable scratchWritable; // Supports keeping a TimestampWritable object without having to import that definition... @@ -71,10 +69,10 @@ public TimestampColumnVector() { public TimestampColumnVector(int len) { super(len); - epochDay = new long[len]; - nanoOfDay = new long[len]; + time = new long[len]; + nanos = new int[len]; - scratchPisaTimestamp = new PisaTimestamp(); + scratchTimestamp = new Timestamp(0); scratchWritable = null; // Allocated by caller. } @@ -84,48 +82,27 @@ public TimestampColumnVector(int len) { * @return */ public int getLength() { - return epochDay.length; + return time.length; } /** - * Returnt a row's epoch day. + * Return a row's Timestamp.getTime() value. * We assume the entry has already been NULL checked and isRepeated adjusted. * @param elementNum * @return */ - public long getEpochDay(int elementNum) { - return epochDay[elementNum]; + public long getTime(int elementNum) { + return time[elementNum]; } /** - * Return a row's nano of day. + * Return a row's Timestamp.getNanos() value. * We assume the entry has already been NULL checked and isRepeated adjusted. * @param elementNum * @return */ - public long getNanoOfDay(int elementNum) { - return nanoOfDay[elementNum]; - } - - /** - * Get a scratch PisaTimestamp object from a row of the column. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return scratch - */ - public PisaTimestamp asScratchPisaTimestamp(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp; - } - - /** - * Set a PisaTimestamp object from a row of the column. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param pisaTimestamp - * @param elementNum - */ - public void pisaTimestampUpdate(PisaTimestamp pisaTimestamp, int elementNum) { - pisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); + public int getNanos(int elementNum) { + return nanos[elementNum]; } /** @@ -135,154 +112,133 @@ public void pisaTimestampUpdate(PisaTimestamp pisaTimestamp, int elementNum) { * @param elementNum */ public void timestampUpdate(Timestamp timestamp, int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - scratchPisaTimestamp.timestampUpdate(timestamp); + timestamp.setTime(time[elementNum]); + timestamp.setNanos(nanos[elementNum]); } /** - * Compare row to PisaTimestamp. + * Return the scratch Timestamp object set from a row. * We assume the entry has already been NULL checked and isRepeated adjusted. * @param elementNum - * @param pisaTimestamp - * @return -1, 0, 1 standard compareTo values. - */ - public int compareTo(int elementNum, PisaTimestamp pisaTimestamp) { - return PisaTimestamp.compareTo(epochDay[elementNum], nanoOfDay[elementNum], pisaTimestamp); - } - - /** - * Compare PisaTimestamp to row. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param pisaTimestamp - * @param elementNum - * @return -1, 0, 1 standard compareTo values. + * @return */ - public int compareTo(PisaTimestamp pisaTimestamp, int elementNum) { - return PisaTimestamp.compareTo(pisaTimestamp, epochDay[elementNum], nanoOfDay[elementNum]); + public Timestamp asScratchTimestamp(int elementNum) { + scratchTimestamp.setTime(time[elementNum]); + scratchTimestamp.setNanos(nanos[elementNum]); + return scratchTimestamp; } /** - * Compare a row to another TimestampColumnVector's row. - * @param elementNum1 - * @param timestampColVector2 - * @param elementNum2 + * Return the scratch timestamp (contents undefined). * @return */ - public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2, - int elementNum2) { - return PisaTimestamp.compareTo( - epochDay[elementNum1], nanoOfDay[elementNum1], - timestampColVector2.epochDay[elementNum2], timestampColVector2.nanoOfDay[elementNum2]); + public Timestamp getScratchTimestamp() { + return scratchTimestamp; } /** - * Compare another TimestampColumnVector's row to a row. - * @param timestampColVector1 - * @param elementNum1 - * @param elementNum2 + * Return a long representation of a Timestamp. + * @param elementNum * @return */ - public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1, - int elementNum2) { - return PisaTimestamp.compareTo( - timestampColVector1.epochDay[elementNum1], timestampColVector1.nanoOfDay[elementNum1], - epochDay[elementNum2], nanoOfDay[elementNum2]); - } - - public void add(PisaTimestamp timestamp1, PisaTimestamp timestamp2, int resultElementNum) { - PisaTimestamp.add(timestamp1, timestamp2, scratchPisaTimestamp); - epochDay[resultElementNum] = scratchPisaTimestamp.getEpochDay(); - nanoOfDay[resultElementNum] = scratchPisaTimestamp.getNanoOfDay(); - } - - public void subtract(PisaTimestamp timestamp1, PisaTimestamp timestamp2, int resultElementNum) { - PisaTimestamp.subtract(timestamp1, timestamp2, scratchPisaTimestamp); - epochDay[resultElementNum] = scratchPisaTimestamp.getEpochDay(); - nanoOfDay[resultElementNum] = scratchPisaTimestamp.getNanoOfDay(); + public long getTimestampAsLong(int elementNum) { + scratchTimestamp.setTime(time[elementNum]); + scratchTimestamp.setNanos(nanos[elementNum]); + return getTimestampAsLong(scratchTimestamp); } /** - * Return row as a double with the integer part as the seconds and the fractional part as - * the nanoseconds the way the Timestamp class does it. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return seconds.nanoseconds + * Return a long representation of a Timestamp. + * @param timestamp + * @return */ - public double getTimestampSecondsWithFractionalNanos(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp.getTimestampSecondsWithFractionalNanos(); + public static long getTimestampAsLong(Timestamp timestamp) { + return millisToSeconds(timestamp.getTime()); } + // Copy of TimestampWritable.millisToSeconds /** - * Return row as integer as the seconds the way the Timestamp class does it. - * We assume the entry has already been NULL checked and isRepeated adjusted. - * @param elementNum - * @return seconds + * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of + * seconds. 500 would round to 0, -500 would round to -1. */ - public long getTimestampSeconds(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp.getTimestampSeconds(); + private static long millisToSeconds(long millis) { + if (millis >= 0) { + return millis / 1000; + } else { + return (millis - 999) / 1000; + } } - /** - * Return row as milliseconds the way the Timestamp class does it. - * We assume the entry has already been NULL checked and isRepeated adjusted. + * Return a double representation of a Timestamp. * @param elementNum * @return */ - public long getTimestampMilliseconds(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp.getTimestampMilliseconds(); + public double getTimestampAsDouble(int elementNum) { + scratchTimestamp.setTime(time[elementNum]); + scratchTimestamp.setNanos(nanos[elementNum]); + return getTimestampAsDouble(scratchTimestamp); } /** - * Return row as epoch seconds. - * We assume the entry has already been NULL checked and isRepeated adjusted. + * Return a double representation of a Timestamp. * @param elementNum * @return */ - public long getEpochSeconds(int elementNum) { - return PisaTimestamp.getEpochSecondsFromEpochDayAndNanoOfDay(epochDay[elementNum], nanoOfDay[elementNum]); + public static double getTimestampAsDouble(Timestamp timestamp) { + // Same algorithm as TimestampWritable (not currently import-able here). + double seconds, nanos; + seconds = millisToSeconds(timestamp.getTime()); + nanos = timestamp.getNanos(); + return seconds + nanos / 1000000000; } /** - * Return row as epoch milliseconds. + * Compare row to Timestamp. * We assume the entry has already been NULL checked and isRepeated adjusted. * @param elementNum - * @return + * @param timestamp + * @return -1, 0, 1 standard compareTo values. */ - public long getEpochMilliseconds(int elementNum) { - return PisaTimestamp.getEpochMillisecondsFromEpochDayAndNanoOfDay(epochDay[elementNum], nanoOfDay[elementNum]); + public int compareTo(int elementNum, Timestamp timestamp) { + return asScratchTimestamp(elementNum).compareTo(timestamp); } /** - * Return row as signed nanos (-999999999 to 999999999). - * NOTE: This is not the same as the Timestamp class nanos (which is always positive). + * Compare Timestamp to row. * We assume the entry has already been NULL checked and isRepeated adjusted. + * @param timestamp * @param elementNum - * @return + * @return -1, 0, 1 standard compareTo values. */ - public int getSignedNanos(int elementNum) { - return PisaTimestamp.getSignedNanos(nanoOfDay[elementNum]); + public int compareTo(Timestamp timestamp, int elementNum) { + return timestamp.compareTo(asScratchTimestamp(elementNum)); } /** - * Get scratch timestamp with value of a row. - * @param elementNum + * Compare a row to another TimestampColumnVector's row. + * @param elementNum1 + * @param timestampColVector2 + * @param elementNum2 * @return */ - public Timestamp asScratchTimestamp(int elementNum) { - scratchPisaTimestamp.update(epochDay[elementNum], nanoOfDay[elementNum]); - return scratchPisaTimestamp.asScratchTimestamp(); + public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2, + int elementNum2) { + return asScratchTimestamp(elementNum1).compareTo( + timestampColVector2.asScratchTimestamp(elementNum2)); } /** - * Get scratch Pisa timestamp for use by the caller. + * Compare another TimestampColumnVector's row to a row. + * @param timestampColVector1 + * @param elementNum1 + * @param elementNum2 * @return */ - public PisaTimestamp useScratchPisaTimestamp() { - return scratchPisaTimestamp; + public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1, + int elementNum2) { + return timestampColVector1.asScratchTimestamp(elementNum1).compareTo( + asScratchTimestamp(elementNum2)); } @Override @@ -290,8 +246,8 @@ public void setElement(int outElementNum, int inputElementNum, ColumnVector inpu TimestampColumnVector timestampColVector = (TimestampColumnVector) inputVector; - epochDay[outElementNum] = timestampColVector.epochDay[inputElementNum]; - nanoOfDay[outElementNum] = timestampColVector.nanoOfDay[inputElementNum]; + time[outElementNum] = timestampColVector.time[inputElementNum]; + nanos[outElementNum] = timestampColVector.nanos[inputElementNum]; } // Simplify vector by brute-force flattening noNulls and isRepeating @@ -301,17 +257,17 @@ public void flatten(boolean selectedInUse, int[] sel, int size) { flattenPush(); if (isRepeating) { isRepeating = false; - long repeatEpochDay = epochDay[0]; - long repeatNanoOfDay = nanoOfDay[0]; + long repeatFastTime = time[0]; + int repeatNanos = nanos[0]; if (selectedInUse) { for (int j = 0; j < size; j++) { int i = sel[j]; - epochDay[i] = repeatEpochDay; - nanoOfDay[i] = repeatNanoOfDay; + time[i] = repeatFastTime; + nanos[i] = repeatNanos; } } else { - Arrays.fill(epochDay, 0, size, repeatEpochDay); - Arrays.fill(nanoOfDay, 0, size, repeatNanoOfDay); + Arrays.fill(time, 0, size, repeatFastTime); + Arrays.fill(nanos, 0, size, repeatNanos); } flattenRepeatingNulls(selectedInUse, sel, size); } @@ -319,71 +275,23 @@ public void flatten(boolean selectedInUse, int[] sel, int size) { } /** - * Set a row from a PisaTimestamp. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param pisaTimestamp - */ - public void set(int elementNum, PisaTimestamp pisaTimestamp) { - this.epochDay[elementNum] = pisaTimestamp.getEpochDay(); - this.nanoOfDay[elementNum] = pisaTimestamp.getNanoOfDay(); - } - - /** * Set a row from a timestamp. * We assume the entry has already been isRepeated adjusted. * @param elementNum * @param timestamp */ public void set(int elementNum, Timestamp timestamp) { - scratchPisaTimestamp.updateFromTimestamp(timestamp); - this.epochDay[elementNum] = scratchPisaTimestamp.getEpochDay(); - this.nanoOfDay[elementNum] = scratchPisaTimestamp.getNanoOfDay(); + this.time[elementNum] = timestamp.getTime(); + this.nanos[elementNum] = timestamp.getNanos(); } /** - * Set a row from a epoch seconds and signed nanos (-999999999 to 999999999). + * Set a row from the current value in the scratch timestamp. * @param elementNum - * @param epochSeconds - * @param signedNanos */ - public void setEpochSecondsAndSignedNanos(int elementNum, long epochSeconds, int signedNanos) { - scratchPisaTimestamp.updateFromEpochSecondsAndSignedNanos(epochSeconds, signedNanos); - set(elementNum, scratchPisaTimestamp); - } - - /** - * Set a row from timestamp milliseconds. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param timestampMilliseconds - */ - public void setTimestampMilliseconds(int elementNum, long timestampMilliseconds) { - scratchPisaTimestamp.updateFromTimestampMilliseconds(timestampMilliseconds); - set(elementNum, scratchPisaTimestamp.useScratchTimestamp()); - } - - /** - * Set a row from timestamp seconds. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param timestamp - */ - public void setTimestampSeconds(int elementNum, long timestampSeconds) { - scratchPisaTimestamp.updateFromTimestampSeconds(timestampSeconds); - set(elementNum, scratchPisaTimestamp); - } - - /** - * Set a row from a double timestamp seconds with fractional nanoseconds. - * We assume the entry has already been isRepeated adjusted. - * @param elementNum - * @param timestamp - */ - public void setTimestampSecondsWithFractionalNanoseconds(int elementNum, - double secondsWithFractionalNanoseconds) { - scratchPisaTimestamp.updateFromTimestampSecondsWithFractionalNanoseconds(secondsWithFractionalNanoseconds); - set(elementNum, scratchPisaTimestamp); + public void setFromScratchTimestamp(int elementNum) { + this.time[elementNum] = scratchTimestamp.getTime(); + this.nanos[elementNum] = scratchTimestamp.getNanos(); } /** @@ -392,8 +300,8 @@ public void setTimestampSecondsWithFractionalNanoseconds(int elementNum, * @param elementNum */ public void setNullValue(int elementNum) { - epochDay[elementNum] = 0; - nanoOfDay[elementNum] = 1; + time[elementNum] = 0; + nanos[elementNum] = 1; } // Copy the current object contents into the output. Only copy selected entries, @@ -407,8 +315,8 @@ public void copySelected( // Handle repeating case if (isRepeating) { - output.epochDay[0] = epochDay[0]; - output.nanoOfDay[0] = nanoOfDay[0]; + output.time[0] = time[0]; + output.nanos[0] = nanos[0]; output.isNull[0] = isNull[0]; output.isRepeating = true; return; @@ -420,13 +328,13 @@ public void copySelected( if (selectedInUse) { for (int j = 0; j < size; j++) { int i = sel[j]; - output.epochDay[i] = epochDay[i]; - output.nanoOfDay[i] = nanoOfDay[i]; + output.time[i] = time[i]; + output.nanos[i] = nanos[i]; } } else { - System.arraycopy(epochDay, 0, output.epochDay, 0, size); - System.arraycopy(nanoOfDay, 0, output.nanoOfDay, 0, size); + System.arraycopy(time, 0, output.time, 0, size); + System.arraycopy(nanos, 0, output.nanos, 0, size); } // Copy nulls over if needed @@ -444,26 +352,14 @@ public void copySelected( } /** - * Fill all the vector entries with a PisaTimestamp. - * @param pisaTimestamp - */ - public void fill(PisaTimestamp pisaTimestamp) { - noNulls = true; - isRepeating = true; - epochDay[0] = pisaTimestamp.getEpochDay(); - nanoOfDay[0] = pisaTimestamp.getNanoOfDay(); - } - - /** * Fill all the vector entries with a timestamp. * @param timestamp */ public void fill(Timestamp timestamp) { noNulls = true; isRepeating = true; - scratchPisaTimestamp.updateFromTimestamp(timestamp); - epochDay[0] = scratchPisaTimestamp.getEpochDay(); - nanoOfDay[0] = scratchPisaTimestamp.getNanoOfDay(); + time[0] = timestamp.getTime(); + nanos[0] = timestamp.getNanos(); } /** @@ -489,8 +385,9 @@ public void stringifyValue(StringBuilder buffer, int row) { row = 0; } if (noNulls || !isNull[row]) { - scratchPisaTimestamp.update(epochDay[row], nanoOfDay[row]); - buffer.append(scratchPisaTimestamp.toString()); + scratchTimestamp.setTime(time[row]); + scratchTimestamp.setNanos(nanos[row]); + buffer.append(scratchTimestamp.toString()); } else { buffer.append("null"); } diff --git storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java new file mode 100644 index 0000000..727c1e6 --- /dev/null +++ storage-api/src/java/org/apache/hive/common/util/IntervalDayTimeUtils.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import java.math.BigDecimal; +import java.text.SimpleDateFormat; + +import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; + + +/** + * DateUtils. Thread-safe class + * + */ +public class IntervalDayTimeUtils { + + private static final ThreadLocal dateFormatLocal = new ThreadLocal() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat("yyyy-MM-dd"); + } + }; + + public static SimpleDateFormat getDateFormat() { + return dateFormatLocal.get(); + } + + public static final int NANOS_PER_SEC = 1000000000; + public static final BigDecimal MAX_INT_BD = new BigDecimal(Integer.MAX_VALUE); + public static final BigDecimal NANOS_PER_SEC_BD = new BigDecimal(NANOS_PER_SEC); + + public static int parseNumericValueWithRange(String fieldName, + String strVal, int minValue, int maxValue) throws IllegalArgumentException { + int result = 0; + if (strVal != null) { + result = Integer.parseInt(strVal); + if (result < minValue || result > maxValue) { + throw new IllegalArgumentException(String.format("%s value %d outside range [%d, %d]", + fieldName, result, minValue, maxValue)); + } + } + return result; + } + + public static long getIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime) { + return intervalDayTime.getTotalSeconds() * NANOS_PER_SEC + intervalDayTime.getNanos(); + } + + public static void setIntervalDayTimeTotalNanos(HiveIntervalDayTime intervalDayTime, + long totalNanos) { + intervalDayTime.set(totalNanos / NANOS_PER_SEC, (int) (totalNanos % NANOS_PER_SEC)); + } + + public static long getIntervalDayTimeTotalSecondsFromTotalNanos(long totalNanos) { + return totalNanos / NANOS_PER_SEC; + } + + public static int getIntervalDayTimeNanosFromTotalNanos(long totalNanos) { + return (int) (totalNanos % NANOS_PER_SEC); + } +} diff --git storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java deleted file mode 100644 index 8e7395c..0000000 --- storage-api/src/test/org/apache/hadoop/hive/common/type/TestPisaTimestamp.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common.type; - -import org.junit.Test; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.sql.Timestamp; -import java.util.Random; -import org.apache.hadoop.hive.common.type.RandomTypeUtil; - -import static org.junit.Assert.*; - -/** - * Test for ListColumnVector - */ -public class TestPisaTimestamp { - - private static int TEST_COUNT = 5000; - - @Test - public void testPisaTimestampCreate() throws Exception { - - Random r = new Random(1234); - - for (int i = 0; i < TEST_COUNT; i++) { - Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); - PisaTimestamp pisaTimestamp = new PisaTimestamp(randTimestamp); - Timestamp reconstructedTimestamp = new Timestamp(0); - pisaTimestamp.timestampUpdate(reconstructedTimestamp); - if (!randTimestamp.equals(reconstructedTimestamp)) { - assertTrue(false); - } - } - } - - static BigDecimal BIG_MAX_LONG = new BigDecimal(Long.MAX_VALUE); - static BigDecimal BIG_MIN_LONG = new BigDecimal(Long.MIN_VALUE); - static BigDecimal BIG_NANOSECONDS_PER_DAY = new BigDecimal(PisaTimestamp.NANOSECONDS_PER_DAY); - - static boolean beyondLongRange = false; - - private BigDecimal[] randomEpochDayAndNanoOfDay(Random r) { - double randDouble = (r.nextDouble() - 0.5D) * 2.0D; - randDouble *= PisaTimestamp.NANOSECONDS_PER_DAY; - randDouble *= 365 * 10000; - BigDecimal bigDecimal = new BigDecimal(randDouble); - bigDecimal = bigDecimal.setScale(0, RoundingMode.HALF_UP); - - if (bigDecimal.compareTo(BIG_MAX_LONG) > 0 || bigDecimal.compareTo(BIG_MIN_LONG) < 0) { - beyondLongRange = true; - } - - BigDecimal[] divideAndRemainder = bigDecimal.divideAndRemainder(BIG_NANOSECONDS_PER_DAY); - - return new BigDecimal[] {divideAndRemainder[0], divideAndRemainder[1], bigDecimal}; - } - - private BigDecimal pisaTimestampToBig(PisaTimestamp pisaTimestamp) { - BigDecimal bigNanoOfDay = new BigDecimal(pisaTimestamp.getNanoOfDay()); - - BigDecimal bigEpochDay = new BigDecimal(pisaTimestamp.getEpochDay()); - BigDecimal result = bigEpochDay.multiply(BIG_NANOSECONDS_PER_DAY); - result = result.add(bigNanoOfDay); - return result; - } - - @Test - public void testPisaTimestampArithmetic() throws Exception { - - Random r = new Random(1234); - - - for (int i = 0; i < TEST_COUNT; i++) { - BigDecimal[] random1 = randomEpochDayAndNanoOfDay(r); - long epochDay1 = random1[0].longValue(); - long nanoOfDay1 = random1[1].longValue(); - PisaTimestamp pisa1 = new PisaTimestamp(epochDay1, nanoOfDay1); - BigDecimal big1 = random1[2]; - - BigDecimal[] random2 = randomEpochDayAndNanoOfDay(r); - long epochDay2 = random2[0].longValue(); - long nanoOfDay2 = random2[1].longValue(); - PisaTimestamp pisa2 = new PisaTimestamp(epochDay2, nanoOfDay2); - BigDecimal big2 = random2[2]; - - BigDecimal expectedBig; - PisaTimestamp pisaResult = new PisaTimestamp(); - if (i % 2 == 0) { - expectedBig = big1.add(big2); - PisaTimestamp.add(pisa1, pisa2, pisaResult); - } else { - expectedBig = big1.add(big2.negate()); - PisaTimestamp.subtract(pisa1, pisa2, pisaResult); - } - BigDecimal resultBig = pisaTimestampToBig(pisaResult); - assertEquals(expectedBig, resultBig); - - } - } -} diff --git storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java new file mode 100644 index 0000000..6e5d5c8 --- /dev/null +++ storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampColumnVector.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector; + +import org.junit.Test; + +import java.io.PrintWriter; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.sql.Timestamp; +import java.util.Date; +import java.util.Random; + +import org.apache.hadoop.hive.common.type.RandomTypeUtil; + +import static org.junit.Assert.*; + +/** + * Test for ListColumnVector + */ +public class TestTimestampColumnVector { + + private static int TEST_COUNT = 5000; + + private static int fake = 0; + + @Test + public void testSaveAndRetrieve() throws Exception { + + Random r = new Random(1234); + TimestampColumnVector timestampColVector = new TimestampColumnVector(); + Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; + + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); + randTimestamps[i] = randTimestamp; + timestampColVector.set(i, randTimestamp); + } + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i); + Timestamp randTimestamp = randTimestamps[i]; + if (!retrievedTimestamp.equals(randTimestamp)) { + assertTrue(false); + } + } + } + + @Test + public void testTimestampCompare() throws Exception { + Random r = new Random(1234); + TimestampColumnVector timestampColVector = new TimestampColumnVector(); + Timestamp[] randTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; + Timestamp[] candTimestamps = new Timestamp[VectorizedRowBatch.DEFAULT_SIZE]; + int[] compareToLeftRights = new int[VectorizedRowBatch.DEFAULT_SIZE]; + int[] compareToRightLefts = new int[VectorizedRowBatch.DEFAULT_SIZE]; + + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); + randTimestamps[i] = randTimestamp; + timestampColVector.set(i, randTimestamp); + Timestamp candTimestamp = RandomTypeUtil.getRandTimestamp(r); + candTimestamps[i] = candTimestamp; + compareToLeftRights[i] = candTimestamp.compareTo(randTimestamp); + compareToRightLefts[i] = randTimestamp.compareTo(candTimestamp); + } + + for (int i = 0; i < VectorizedRowBatch.DEFAULT_SIZE; i++) { + Timestamp retrievedTimestamp = timestampColVector.asScratchTimestamp(i); + Timestamp randTimestamp = randTimestamps[i]; + if (!retrievedTimestamp.equals(randTimestamp)) { + assertTrue(false); + } + Timestamp candTimestamp = candTimestamps[i]; + int compareToLeftRight = timestampColVector.compareTo(candTimestamp, i); + if (compareToLeftRight != compareToLeftRights[i]) { + assertTrue(false); + } + int compareToRightLeft = timestampColVector.compareTo(i, candTimestamp); + if (compareToRightLeft != compareToRightLefts[i]) { + assertTrue(false); + } + } + } + + /* + @Test + public void testGenerate() throws Exception { + PrintWriter writer = new PrintWriter("/Users/you/timestamps.txt"); + Random r = new Random(18485); + for (int i = 0; i < 25; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r); + writer.println(randTimestamp.toString()); + } + for (int i = 0; i < 25; i++) { + Timestamp randTimestamp = RandomTypeUtil.getRandTimestamp(r, 1965, 2025); + writer.println(randTimestamp.toString()); + } + writer.close(); + } + */ +}