diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 3cdedba..b3cee11 100644 --- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -100,6 +100,7 @@ public CliDriver() { } public int processCmd(String cmd) { + CliSessionState ss = (CliSessionState) SessionState.get(); ss.setLastCommand(cmd); // Flush the print stream, so it doesn't include output from the last command @@ -169,6 +170,7 @@ public int processCmd(String cmd) { } } } else if (ss.isRemoteMode()) { // remote mode -- connecting to remote hive server + HiveClient client = ss.getClient(); PrintStream out = ss.out; PrintStream err = ss.err; @@ -245,6 +247,7 @@ private String getFirstCmd(String cmd, int length) { } int processLocalCmd(String cmd, CommandProcessor proc, CliSessionState ss) { + int tryCount = 0; boolean needRetry; int ret = 0; diff --git a/conf/hive-default.xml.template b/conf/hive-default.xml.template index ba5b8a9..653f5cc 100644 --- a/conf/hive-default.xml.template +++ b/conf/hive-default.xml.template @@ -1,5 +1,7 @@ - +--> + diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 130fd67..29fee24 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -1473,8 +1473,8 @@ public void testColumnStatistics() throws Throwable { assertEquals(colStats2.getColName(), colName[0]); assertEquals(colStats2.getStatsData().getDoubleStats().getLowValue(), lowValue); assertEquals(colStats2.getStatsData().getDoubleStats().getHighValue(), highValue); - assertEquals(colStats2.getStatsData().getDoubleStats().getNumNulls(), numNulls); - assertEquals(colStats2.getStatsData().getDoubleStats().getNumDVs(), numDVs); + assertEquals(colStats2.getStatsData().getDoubleStats().getNumNulls().longValue(), numNulls); + assertEquals(colStats2.getStatsData().getDoubleStats().getNumDVs().longValue(), numDVs); // test delete column stats; if no col name is passed all column stats associated with the // table is deleted @@ -1531,10 +1531,10 @@ public void testColumnStatistics() throws Throwable { assertNotNull(colStats2); assertEquals(colStats.getStatsDesc().getPartName(), partName); assertEquals(colStats2.getColName(), colName[1]); - assertEquals(colStats2.getStatsData().getStringStats().getMaxColLen(), maxColLen); + assertEquals(colStats2.getStatsData().getStringStats().getMaxColLen().longValue(), maxColLen); assertEquals(colStats2.getStatsData().getStringStats().getAvgColLen(), avgColLen); - assertEquals(colStats2.getStatsData().getStringStats().getNumNulls(), numNulls); - assertEquals(colStats2.getStatsData().getStringStats().getNumDVs(), numDVs); + assertEquals(colStats2.getStatsData().getStringStats().getNumNulls().longValue(), numNulls); + assertEquals(colStats2.getStatsData().getStringStats().getNumDVs().longValue(), numDVs); // test stats deletion at partition level client.deletePartitionColumnStatistics(dbName, tblName, partName, colName[1]); diff --git a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BinaryColumnStatsData.java b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BinaryColumnStatsData.java index 1516b25..b6ee0dd 100644 --- a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BinaryColumnStatsData.java +++ b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BinaryColumnStatsData.java @@ -164,15 +164,18 @@ public BinaryColumnStatsData deepCopy() { @Override public void clear() { setMaxColLenIsSet(false); - this.maxColLen = 0; +// this.maxColLen = 0; setAvgColLenIsSet(false); - this.avgColLen = 0.0; +// this.avgColLen = 0.0; setNumNullsIsSet(false); - this.numNulls = 0; +// this.numNulls = 0; } - public long getMaxColLen() { + public Long getMaxColLen() { + if (isSetMaxColLen()) return this.maxColLen; + else + return null; } public void setMaxColLen(long maxColLen) { @@ -193,8 +196,11 @@ public void setMaxColLenIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAXCOLLEN_ISSET_ID, value); } - public double getAvgColLen() { + public Double getAvgColLen() { + if (isSetAvgColLen()) return this.avgColLen; + else + return null; } public void setAvgColLen(double avgColLen) { @@ -215,8 +221,11 @@ public void setAvgColLenIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __AVGCOLLEN_ISSET_ID, value); } - public long getNumNulls() { + public Long getNumNulls() { + if (isSetNumNulls()) return this.numNulls; + else + return null; } public void setNumNulls(long numNulls) { diff --git a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BooleanColumnStatsData.java b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BooleanColumnStatsData.java index 9ef9c0f..1560116 100644 --- a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BooleanColumnStatsData.java +++ b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BooleanColumnStatsData.java @@ -164,15 +164,18 @@ public BooleanColumnStatsData deepCopy() { @Override public void clear() { setNumTruesIsSet(false); - this.numTrues = 0; +// this.numTrues = 0; setNumFalsesIsSet(false); - this.numFalses = 0; +// this.numFalses = 0; setNumNullsIsSet(false); - this.numNulls = 0; +// this.numNulls = 0; } - public long getNumTrues() { + public Long getNumTrues() { + if (isSetNumTrues()) return this.numTrues; + else + return null; } public void setNumTrues(long numTrues) { @@ -193,8 +196,11 @@ public void setNumTruesIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __NUMTRUES_ISSET_ID, value); } - public long getNumFalses() { + public Long getNumFalses() { + if (isSetNumFalses()) return this.numFalses; + else + return null; } public void setNumFalses(long numFalses) { @@ -215,8 +221,11 @@ public void setNumFalsesIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __NUMFALSES_ISSET_ID, value); } - public long getNumNulls() { + public Long getNumNulls() { + if (isSetNumNulls()) return this.numNulls; + else + return null; } public void setNumNulls(long numNulls) { diff --git a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DecimalColumnStatsData.java b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DecimalColumnStatsData.java index 951d479..d9df883 100644 --- a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DecimalColumnStatsData.java +++ b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DecimalColumnStatsData.java @@ -175,9 +175,9 @@ public void clear() { this.lowValue = null; this.highValue = null; setNumNullsIsSet(false); - this.numNulls = 0; +// this.numNulls = 0; setNumDVsIsSet(false); - this.numDVs = 0; +// this.numDVs = 0; } public Decimal getLowValue() { @@ -226,8 +226,11 @@ public void setHighValueIsSet(boolean value) { } } - public long getNumNulls() { + public Long getNumNulls() { + if (isSetNumNulls()) return this.numNulls; + else + return null; } public void setNumNulls(long numNulls) { @@ -248,8 +251,11 @@ public void setNumNullsIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __NUMNULLS_ISSET_ID, value); } - public long getNumDVs() { + public Long getNumDVs() { + if (isSetNumDVs()) return this.numDVs; + else + return null; } public void setNumDVs(long numDVs) { diff --git a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DoubleColumnStatsData.java b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DoubleColumnStatsData.java index 4203fd8..4867bb6 100644 --- a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DoubleColumnStatsData.java +++ b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/DoubleColumnStatsData.java @@ -171,17 +171,20 @@ public DoubleColumnStatsData deepCopy() { @Override public void clear() { setLowValueIsSet(false); - this.lowValue = 0.0; +// this.lowValue = 0.0; setHighValueIsSet(false); - this.highValue = 0.0; +// this.highValue = 0.0; setNumNullsIsSet(false); - this.numNulls = 0; +// this.numNulls = 0; setNumDVsIsSet(false); - this.numDVs = 0; +// this.numDVs = 0; } - public double getLowValue() { + public Double getLowValue() { + if (isSetLowValue()) return this.lowValue; + else + return null; } public void setLowValue(double lowValue) { @@ -202,8 +205,11 @@ public void setLowValueIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __LOWVALUE_ISSET_ID, value); } - public double getHighValue() { + public Double getHighValue() { + if (isSetHighValue()) return this.highValue; + else + return null; } public void setHighValue(double highValue) { @@ -224,8 +230,11 @@ public void setHighValueIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __HIGHVALUE_ISSET_ID, value); } - public long getNumNulls() { + public Long getNumNulls() { + if (isSetNumNulls()) return this.numNulls; + else + return null; } public void setNumNulls(long numNulls) { @@ -246,8 +255,11 @@ public void setNumNullsIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __NUMNULLS_ISSET_ID, value); } - public long getNumDVs() { + public Long getNumDVs() { + if (isSetNumDVs()) return this.numDVs; + else + return null; } public void setNumDVs(long numDVs) { diff --git a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LongColumnStatsData.java b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LongColumnStatsData.java index d817d46..0b3a7bd 100644 --- a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LongColumnStatsData.java +++ b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/LongColumnStatsData.java @@ -171,17 +171,20 @@ public LongColumnStatsData deepCopy() { @Override public void clear() { setLowValueIsSet(false); - this.lowValue = 0; +// this.lowValue = 0; setHighValueIsSet(false); - this.highValue = 0; +// this.highValue = 0; setNumNullsIsSet(false); - this.numNulls = 0; +// this.numNulls = 0; setNumDVsIsSet(false); - this.numDVs = 0; +// this.numDVs = 0; } - public long getLowValue() { + public Long getLowValue() { + if (isSetLowValue()) return this.lowValue; + else + return null; } public void setLowValue(long lowValue) { @@ -202,8 +205,11 @@ public void setLowValueIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __LOWVALUE_ISSET_ID, value); } - public long getHighValue() { + public Long getHighValue() { + if (isSetHighValue()) return this.highValue; + else + return null; } public void setHighValue(long highValue) { @@ -224,8 +230,11 @@ public void setHighValueIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __HIGHVALUE_ISSET_ID, value); } - public long getNumNulls() { + public Long getNumNulls() { + if (isSetNumNulls()) return this.numNulls; + else + return null; } public void setNumNulls(long numNulls) { @@ -246,8 +255,11 @@ public void setNumNullsIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __NUMNULLS_ISSET_ID, value); } - public long getNumDVs() { + public Long getNumDVs() { + if (isSetNumDVs()) return this.numDVs; + else + return null; } public void setNumDVs(long numDVs) { diff --git a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StringColumnStatsData.java b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StringColumnStatsData.java index db3274a..59b086b 100644 --- a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StringColumnStatsData.java +++ b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StringColumnStatsData.java @@ -176,17 +176,20 @@ public StringColumnStatsData deepCopy() { @Override public void clear() { setMaxColLenIsSet(false); - this.maxColLen = 0; +// this.maxColLen = 0; setAvgColLenIsSet(false); - this.avgColLen = 0.0; +// this.avgColLen = 0.0; setNumNullsIsSet(false); - this.numNulls = 0; +// this.numNulls = 0; setNumDVsIsSet(false); - this.numDVs = 0; +// this.numDVs = 0; } - public long getMaxColLen() { + public Long getMaxColLen() { + if (isSetMaxColLen()) return this.maxColLen; + else + return null; } public void setMaxColLen(long maxColLen) { @@ -207,8 +210,11 @@ public void setMaxColLenIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAXCOLLEN_ISSET_ID, value); } - public double getAvgColLen() { + public Double getAvgColLen() { + if (isSetAvgColLen()) return this.avgColLen; + else + return null; } public void setAvgColLen(double avgColLen) { @@ -229,8 +235,11 @@ public void setAvgColLenIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __AVGCOLLEN_ISSET_ID, value); } - public long getNumNulls() { + public Long getNumNulls() { + if (isSetNumNulls()) return this.numNulls; + else + return null; } public void setNumNulls(long numNulls) { @@ -251,8 +260,11 @@ public void setNumNullsIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __NUMNULLS_ISSET_ID, value); } - public long getNumDVs() { + public Long getNumDVs() { + if (isSetNumDVs()) return this.numDVs; + else + return null; } public void setNumDVs(long numDVs) { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 32da869..f924fb6 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -1230,6 +1230,8 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics statsObj) new TableStatsRequest(dbName, tableName, colNames)).getTableStats(); } + + /** {@inheritDoc} */ @Override public Map> getPartitionColumnStatistics( diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java b/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java index c3e2820..e54c6d3 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java @@ -95,6 +95,7 @@ public static MTableColumnStatistics convertToMTableColumnStatistics(MTable tabl public static void setFieldsIntoOldStats( MTableColumnStatistics mStatsObj, MTableColumnStatistics oldStatsObj) { + if (mStatsObj.getAvgColLen() != null) oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen()); if (mStatsObj.getLongHighValue() != null) { oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue()); @@ -114,28 +115,45 @@ public static void setFieldsIntoOldStats( if (mStatsObj.getDecimalHighValue() != null) { oldStatsObj.setDecimalHighValue(mStatsObj.getDecimalHighValue()); } + if (mStatsObj.getMaxColLen() != null) oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen()); + if (mStatsObj.getNumDVs() != null) oldStatsObj.setNumDVs(mStatsObj.getNumDVs()); + if (mStatsObj.getNumFalses() != null) oldStatsObj.setNumFalses(mStatsObj.getNumFalses()); + if (mStatsObj.getNumTrues() != null) oldStatsObj.setNumTrues(mStatsObj.getNumTrues()); + if (mStatsObj.getNumNulls() != null) oldStatsObj.setNumNulls(mStatsObj.getNumNulls()); oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed()); } public static void setFieldsIntoOldStats( MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStatsObj) { + if (mStatsObj.getAvgColLen() != null) oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen()); + if (mStatsObj.getLongHighValue() != null) oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue()); + if (mStatsObj.getDoubleHighValue() != null) oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue()); oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed()); + if (mStatsObj.getLongLowValue() != null) oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue()); + if (mStatsObj.getDoubleLowValue() != null) oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue()); + if (mStatsObj.getDecimalLowValue() != null) oldStatsObj.setDecimalLowValue(mStatsObj.getDecimalLowValue()); + if (mStatsObj.getDecimalHighValue() != null) oldStatsObj.setDecimalHighValue(mStatsObj.getDecimalHighValue()); + if (mStatsObj.getMaxColLen() != null) oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen()); + if (mStatsObj.getNumDVs() != null) oldStatsObj.setNumDVs(mStatsObj.getNumDVs()); + if (mStatsObj.getNumFalses() != null) oldStatsObj.setNumFalses(mStatsObj.getNumFalses()); + if (mStatsObj.getNumTrues() != null) oldStatsObj.setNumTrues(mStatsObj.getNumTrues()); + if (mStatsObj.getNumNulls() != null) oldStatsObj.setNumNulls(mStatsObj.getNumNulls()); } diff --git a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java index 89c31dc..1666dc3 100644 --- a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java +++ b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java @@ -46,12 +46,12 @@ private Double doubleHighValue; private String decimalLowValue; private String decimalHighValue; - private long numNulls; - private long numDVs; - private double avgColLen; - private long maxColLen; - private long numTrues; - private long numFalses; + private Long numNulls; + private Long numDVs; + private Double avgColLen; + private Long maxColLen; + private Long numTrues; + private Long numFalses; private long lastAnalyzed; public MPartitionColumnStatistics() {} @@ -72,7 +72,7 @@ public void setColName(String colName) { this.colName = colName; } - public long getNumNulls() { + public Long getNumNulls() { return numNulls; } @@ -80,7 +80,7 @@ public void setNumNulls(long numNulls) { this.numNulls = numNulls; } - public long getNumDVs() { + public Long getNumDVs() { return numDVs; } @@ -88,7 +88,7 @@ public void setNumDVs(long numDVs) { this.numDVs = numDVs; } - public double getAvgColLen() { + public Double getAvgColLen() { return avgColLen; } @@ -96,7 +96,7 @@ public void setAvgColLen(double avgColLen) { this.avgColLen = avgColLen; } - public long getMaxColLen() { + public Long getMaxColLen() { return maxColLen; } @@ -104,7 +104,7 @@ public void setMaxColLen(long maxColLen) { this.maxColLen = maxColLen; } - public long getNumTrues() { + public Long getNumTrues() { return numTrues; } @@ -112,7 +112,7 @@ public void setNumTrues(long numTrues) { this.numTrues = numTrues; } - public long getNumFalses() { + public Long getNumFalses() { return numFalses; } @@ -160,20 +160,20 @@ public void setColType(String colType) { this.colType = colType; } - public void setBooleanStats(long numTrues, long numFalses, long numNulls) { + public void setBooleanStats(Long numTrues, Long numFalses, Long numNulls) { this.numTrues = numTrues; this.numFalses = numFalses; this.numNulls = numNulls; } - public void setLongStats(long numNulls, long numNDVs, Long lowValue, Long highValue) { + public void setLongStats(Long numNulls, Long numNDVs, Long lowValue, Long highValue) { this.numNulls = numNulls; this.numDVs = numNDVs; this.longLowValue = lowValue; this.longHighValue = highValue; } - public void setDoubleStats(long numNulls, long numNDVs, Double lowValue, Double highValue) { + public void setDoubleStats(Long numNulls, Long numNDVs, Double lowValue, Double highValue) { this.numNulls = numNulls; this.numDVs = numNDVs; this.doubleLowValue = lowValue; @@ -181,21 +181,21 @@ public void setDoubleStats(long numNulls, long numNDVs, Double lowValue, Double } public void setDecimalStats( - long numNulls, long numNDVs, String lowValue, String highValue) { + Long numNulls, Long numNDVs, String lowValue, String highValue) { this.numNulls = numNulls; this.numDVs = numNDVs; this.decimalLowValue = lowValue; this.decimalHighValue = highValue; } - public void setStringStats(long numNulls, long numNDVs, long maxColLen, double avgColLen) { + public void setStringStats(Long numNulls, Long numNDVs, Long maxColLen, Double avgColLen) { this.numNulls = numNulls; this.numDVs = numNDVs; this.maxColLen = maxColLen; this.avgColLen = avgColLen; } - public void setBinaryStats(long numNulls, long maxColLen, double avgColLen) { + public void setBinaryStats(Long numNulls, Long maxColLen, Double avgColLen) { this.numNulls = numNulls; this.maxColLen = maxColLen; this.avgColLen = avgColLen; diff --git a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java index 44bbab5..bce9f0f 100644 --- a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java +++ b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java @@ -44,12 +44,12 @@ private Double doubleHighValue; private String decimalLowValue; private String decimalHighValue; - private long numNulls; - private long numDVs; - private double avgColLen; - private long maxColLen; - private long numTrues; - private long numFalses; + private Long numNulls; + private Long numDVs; + private Double avgColLen; + private Long maxColLen; + private Long numTrues; + private Long numFalses; private long lastAnalyzed; public MTableColumnStatistics() {} @@ -86,7 +86,7 @@ public void setColType(String colType) { this.colType = colType; } - public long getNumNulls() { + public Long getNumNulls() { return numNulls; } @@ -94,7 +94,7 @@ public void setNumNulls(long numNulls) { this.numNulls = numNulls; } - public long getNumDVs() { + public Long getNumDVs() { return numDVs; } @@ -102,7 +102,7 @@ public void setNumDVs(long numDVs) { this.numDVs = numDVs; } - public double getAvgColLen() { + public Double getAvgColLen() { return avgColLen; } @@ -110,7 +110,7 @@ public void setAvgColLen(double avgColLen) { this.avgColLen = avgColLen; } - public long getMaxColLen() { + public Long getMaxColLen() { return maxColLen; } @@ -118,7 +118,7 @@ public void setMaxColLen(long maxColLen) { this.maxColLen = maxColLen; } - public long getNumTrues() { + public Long getNumTrues() { return numTrues; } @@ -126,7 +126,7 @@ public void setNumTrues(long numTrues) { this.numTrues = numTrues; } - public long getNumFalses() { + public Long getNumFalses() { return numFalses; } @@ -150,20 +150,20 @@ public void setDbName(String dbName) { this.dbName = dbName; } - public void setBooleanStats(long numTrues, long numFalses, long numNulls) { + public void setBooleanStats(Long numTrues, Long numFalses, Long numNulls) { this.numTrues = numTrues; this.numFalses = numFalses; this.numNulls = numNulls; } - public void setLongStats(long numNulls, long numNDVs, Long lowValue, Long highValue) { + public void setLongStats(Long numNulls, Long numNDVs, Long lowValue, Long highValue) { this.numNulls = numNulls; this.numDVs = numNDVs; this.longLowValue = lowValue; this.longHighValue = highValue; } - public void setDoubleStats(long numNulls, long numNDVs, Double lowValue, Double highValue) { + public void setDoubleStats(Long numNulls, Long numNDVs, Double lowValue, Double highValue) { this.numNulls = numNulls; this.numDVs = numNDVs; this.doubleLowValue = lowValue; @@ -171,21 +171,21 @@ public void setDoubleStats(long numNulls, long numNDVs, Double lowValue, Double } public void setDecimalStats( - long numNulls, long numNDVs, String lowValue, String highValue) { + Long numNulls, Long numNDVs, String lowValue, String highValue) { this.numNulls = numNulls; this.numDVs = numNDVs; this.decimalLowValue = lowValue; this.decimalHighValue = highValue; } - public void setStringStats(long numNulls, long numNDVs, long maxColLen, double avgColLen) { + public void setStringStats(Long numNulls, Long numNDVs, Long maxColLen, Double avgColLen) { this.numNulls = numNulls; this.numDVs = numNDVs; this.maxColLen = maxColLen; this.avgColLen = avgColLen; } - public void setBinaryStats(long numNulls, long maxColLen, double avgColLen) { + public void setBinaryStats(Long numNulls, Long maxColLen, Double avgColLen) { this.numNulls = numNulls; this.maxColLen = maxColLen; this.avgColLen = avgColLen; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java new file mode 100644 index 0000000..b124907 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java @@ -0,0 +1,321 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import java.io.IOException; +import java.io.Serializable; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.StatObjectConverter; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; +import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Decimal; +import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; +import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; +import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; +import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; +import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; +import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.util.StringUtils; + +/** + * ColumnStatsTask implementation. + **/ + +public class ColumnStatsUpdateTask extends Task + implements Serializable { + private static final long serialVersionUID = 1L; + private static transient final Log LOG = LogFactory + .getLog(ColumnStatsUpdateTask.class); + + public ColumnStatsUpdateTask() { + super(); + } + + @Override + public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext ctx) { + super.initialize(conf, queryPlan, ctx); + try { + JobConf job = new JobConf(conf); + } catch (Exception e) { + LOG.error(StringUtils.stringifyException(e)); + throw new RuntimeException(e); + } + } + + private ColumnStatistics constructColumnStatsFromInput() { + + String dbName = SessionState.get().getCurrentDatabase(); + ColumnStatsDesc desc = work.getColStats(); + String tableName = desc.getTableName(); + String partName = work.getPartName(); + List colName = desc.getColName(); + List colType = desc.getColType(); + + ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); + + statsObj.setColName(colName.get(0)); + + statsObj.setColType(colType.get(0)); + + ColumnStatisticsData statsData = new ColumnStatisticsData(); + + String s = colType.get(0); + + if (s.equalsIgnoreCase("long")) { + // private long lowValue; // optional + // private long highValue; // optional + // private long numNulls; // required + // private long numDVs; // required + LongColumnStatsData longStats = new LongColumnStatsData(); + longStats.clear(); + HashMap mapProp = work.getMapProp(); + for (Entry entry : mapProp.entrySet()) { + String fName = entry.getKey(); + String value = entry.getValue(); + if (fName.equals("numNulls")) { + longStats.setNumNulls(Long.parseLong(value)); + } else if (fName.equals("numDVs")) { + longStats.setNumDVs(Long.parseLong(value)); + } else if (fName.equals("lowValue")) { + longStats.setLowValue(Long.parseLong(value)); + } else if (fName.equals("highValue")) { + longStats.setHighValue(Long.parseLong(value)); + } + } + statsData.setLongStats(longStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("double")) { + // private double lowValue; // optional + // private double highValue; // optional + // private long numNulls; // required + // private long numDVs; // required + DoubleColumnStatsData doubleStats = new DoubleColumnStatsData(); + doubleStats.clear(); + HashMap mapProp = work.getMapProp(); + for (Entry entry : mapProp.entrySet()) { + String fName = entry.getKey(); + String value = entry.getValue(); + if (fName.equals("numNulls")) { + doubleStats.setNumNulls(Long.parseLong(value)); + } else if (fName.equals("numDVs")) { + doubleStats.setNumDVs(Long.parseLong(value)); + } else if (fName.equals("lowValue")) { + doubleStats.setLowValue(Double.parseDouble(value)); + } else if (fName.equals("highValue")) { + doubleStats.setHighValue(Double.parseDouble(value)); + } + } + statsData.setDoubleStats(doubleStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("string")) { + // private long maxColLen; // required + // private double avgColLen; // required + // private long numNulls; // required + // private long numDVs; // required + StringColumnStatsData stringStats = new StringColumnStatsData(); + stringStats.clear(); + HashMap mapProp = work.getMapProp(); + for (Entry entry : mapProp.entrySet()) { + String fName = entry.getKey(); + String value = entry.getValue(); + if (fName.equals("numNulls")) { + stringStats.setNumNulls(Long.parseLong(value)); + } else if (fName.equals("numDVs")) { + stringStats.setNumDVs(Long.parseLong(value)); + } else if (fName.equals("avgColLen")) { + stringStats.setAvgColLen(Double.parseDouble(value)); + } else if (fName.equals("maxColLen")) { + stringStats.setMaxColLen(Long.parseLong(value)); + } + } + statsData.setStringStats(stringStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("boolean")) { + // private long numTrues; // required + // private long numFalses; // required + // private long numNulls; // required + BooleanColumnStatsData booleanStats = new BooleanColumnStatsData(); + booleanStats.clear(); + HashMap mapProp = work.getMapProp(); + for (Entry entry : mapProp.entrySet()) { + String fName = entry.getKey(); + String value = entry.getValue(); + if (fName.equals("numNulls")) { + booleanStats.setNumNulls(Long.parseLong(value)); + } else if (fName.equals("numTrues")) { + booleanStats.setNumTrues(Long.parseLong(value)); + } else if (fName.equals("numFalses")) { + booleanStats.setNumFalses(Long.parseLong(value)); + } + } + statsData.setBooleanStats(booleanStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("binary")) { + // private long maxColLen; // required + // private double avgColLen; // required + // private long numNulls; // required + BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); + binaryStats.clear(); + HashMap mapProp = work.getMapProp(); + for (Entry entry : mapProp.entrySet()) { + String fName = entry.getKey(); + String value = entry.getValue(); + if (fName.equals("numNulls")) { + binaryStats.setNumNulls(Long.parseLong(value)); + } else if (fName.equals("avgColLen")) { + binaryStats.setAvgColLen(Double.parseDouble(value)); + } else if (fName.equals("maxColLen")) { + binaryStats.setMaxColLen(Long.parseLong(value)); + } + } + statsData.setBinaryStats(binaryStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("decimal")) { + // private Decimal lowValue; // optional + // private Decimal highValue; // optional + // private long numNulls; // required + // private long numDVs; // required + DecimalColumnStatsData decimalStats = new DecimalColumnStatsData(); + decimalStats.clear(); + HashMap mapProp = work.getMapProp(); + for (Entry entry : mapProp.entrySet()) { + String fName = entry.getKey(); + String value = entry.getValue(); + if (fName.equals("numNulls")) { + decimalStats.setNumNulls(Long.parseLong(value)); + } else if (fName.equals("numDVs")) { + decimalStats.setNumDVs(Long.parseLong(value)); + } else if (fName.equals("lowValue")) { + BigDecimal d = new BigDecimal(value); + decimalStats.setLowValue(new Decimal(ByteBuffer.wrap(d.unscaledValue().toByteArray()), + (short) d.scale())); + } else if (fName.equals("highValue")) { + BigDecimal d = new BigDecimal(value); + decimalStats.setHighValue(new Decimal(ByteBuffer.wrap(d.unscaledValue().toByteArray()), + (short) d.scale())); + } + } + statsData.setDecimalStats(decimalStats); + statsObj.setStatsData(statsData); + } + + ColumnStatisticsDesc statsDesc = getColumnStatsDesc(dbName, tableName, + partName, partName == null); + ColumnStatistics colStats = new ColumnStatistics(); + colStats.setStatsDesc(statsDesc); + colStats.addToStatsObj(statsObj); + return colStats; + } + + private ColumnStatisticsDesc getColumnStatsDesc(String dbName, + String tableName, String partName, boolean isTblLevel) { + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); + statsDesc.setDbName(dbName); + statsDesc.setTableName(tableName); + statsDesc.setIsTblLevel(isTblLevel); + if (!isTblLevel) { + statsDesc.setPartName(partName); + } else { + statsDesc.setPartName(null); + } + return statsDesc; + } + + private int persistTableStats() throws HiveException, MetaException, + IOException { + // Fetch result of the analyze table .. compute statistics for columns + // .. + // Construct a column statistics object from the result + ColumnStatistics colStats = constructColumnStatsFromInput(); + // Persist the column statistics object to the metastore + db.updateTableColumnStatistics(colStats); + return 0; + } + + private int persistPartitionStats() throws HiveException, MetaException, + IOException { + + // Fetch result of the analyze table partition (p1=c1).. compute + // statistics for columns .. + // Construct a column statistics object from the result + ColumnStatistics colStats = constructColumnStatsFromInput(); + // Persist the column statistics object to the metastore + db.updatePartitionColumnStatistics(colStats); + return 0; + } + + @Override + public int execute(DriverContext driverContext) { + try { + if (work.getColStats().isTblLevel()) { + return persistTableStats(); + } else { + return persistPartitionStats(); + } + } catch (Exception e) { + LOG.info(e); + } + return 1; + } + + @Override + public StageType getType() { + return StageType.COLUMNSTATS; + } + + @Override + public String getName() { + return "COLUMNSTATS TASK"; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java index ad6e19c..e1e3bab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork; import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanTask; import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork; +import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; import org.apache.hadoop.hive.ql.plan.ConditionalWork; import org.apache.hadoop.hive.ql.plan.CopyWork; @@ -92,6 +93,7 @@ public TaskTuple(Class workClass, Class> taskClass) { StatsTask.class)); taskvec.add(new TaskTuple(StatsNoJobWork.class, StatsNoJobTask.class)); taskvec.add(new TaskTuple(ColumnStatsWork.class, ColumnStatsTask.class)); + taskvec.add(new TaskTuple(ColumnStatsUpdateWork.class, ColumnStatsUpdateTask.class)); taskvec.add(new TaskTuple(MergeWork.class, BlockMergeTask.class)); taskvec.add(new TaskTuple(DependencyCollectionWork.class, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java index e83bc17..2b746ff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java @@ -112,6 +112,7 @@ public int execute(DriverContext driverContext) { LOG.debug("Task: " + getId() + ", Summary: " + totalInputFileSize + "," + totalInputNumFiles + "," + numReducers); + } String reason = MapRedTask.isEligibleForLocalMode(conf, numReducers, @@ -121,9 +122,11 @@ public int execute(DriverContext driverContext) { cloneConf(); ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, "local"); console.printInfo("Selecting local mode for task: " + getId()); + this.setLocalMode(true); } else { console.printInfo("Cannot run job locally: " + reason); + this.setLocalMode(false); } } @@ -173,6 +176,7 @@ public int execute(DriverContext driverContext) { OutputStream out = FileSystem.getLocal(conf).create(planPath); MapredWork plan = getWork(); LOG.info("Generating plan file " + planPath.toString()); + Utilities.serializePlan(plan, out, conf); String isSilent = "true".equalsIgnoreCase(System @@ -206,6 +210,7 @@ public int execute(DriverContext driverContext) { LOG.info("Executing: " + cmdLine); + // Inherit Java system variables String hadoopOpts; StringBuilder sb = new StringBuilder(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 37b1669..5caab45 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -14,7 +14,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.conf.HiveConf;; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index c0322fb..8ca1595 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -25,6 +25,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -57,6 +58,7 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ArchiveUtils; +import org.apache.hadoop.hive.ql.exec.ColumnStatsUpdateTask; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Task; @@ -90,6 +92,8 @@ import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.AlterTableExchangePartition; import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc; +import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; +import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc; import org.apache.hadoop.hive.ql.plan.CreateIndexDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -275,6 +279,8 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { analyzeAlterTableClusterSort(ast, tableName, partSpec); } else if (ast.getToken().getType() == HiveParser.TOK_COMPACT) { analyzeAlterTableCompact(ast, tableName, partSpec); + } else if(ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS){ + analyzeAlterTableUpdateStats(ast,tblPart); } break; } @@ -377,6 +383,9 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { case HiveParser.TOK_ALTERTABLE_RENAME: analyzeAlterTableRename(ast, false); break; + case HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS: + analyzeAlterTableUpdateStats(ast); + break; case HiveParser.TOK_ALTERTABLE_TOUCH: analyzeAlterTableTouch(ast); break; @@ -506,6 +515,76 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { } } + private void analyzeAlterTableUpdateStats(ASTNode ast) + throws SemanticException { + String tblName = getUnescapedName((ASTNode) ast.getChild(0)); + String colName = getUnescapedName((ASTNode) ast.getChild(1)); + HashMap mapProp = getProps((ASTNode) (ast.getChild(2)) + .getChild(0)); + + Table tbl = null; + try { + tbl = db.getTable(tblName); + } catch (HiveException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + String colType = null; + List cols = tbl.getCols(); + for (FieldSchema col : cols) { + if (colName.equalsIgnoreCase(col.getName())) { + colType = col.getType(); + break; + } + } + + ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tblName, + Arrays.asList(new String[] { colName }), + Arrays.asList(new String[] { colType }), true); + ColumnStatsUpdateTask cStatsUpdateTask = (ColumnStatsUpdateTask) TaskFactory + .get(new ColumnStatsUpdateWork(cStatsDesc, null, mapProp), conf); + rootTasks.add(cStatsUpdateTask); + } + + private void analyzeAlterTableUpdateStats(ASTNode ast,TablePartition tblPart) + throws SemanticException { + String tblName = tblPart.tableName; + HashMap partSpec = tblPart.partSpec; + assert partSpec.size()==1; + String partName = null; + for(Entry entry : partSpec.entrySet()){ + partName = entry.toString(); + } + String colName = getUnescapedName((ASTNode) ast.getChild(0)); + HashMap mapProp = getProps((ASTNode) (ast.getChild(1)) + .getChild(0)); + + Table tbl = null; + try { + tbl = db.getTable(tblName); + } catch (HiveException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + String colType = null; + List cols = tbl.getCols(); + for (FieldSchema col : cols) { + if (colName.equalsIgnoreCase(col.getName())) { + colType = col.getType(); + break; + } + } + + ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tblName, + Arrays.asList(new String[] { colName }), + Arrays.asList(new String[] { colType }), false); + ColumnStatsUpdateTask cStatsUpdateTask = (ColumnStatsUpdateTask) TaskFactory + .get(new ColumnStatsUpdateWork(cStatsDesc, partName, mapProp), conf); + rootTasks.add(cStatsUpdateTask); + } + private void analyzeSetShowRole(ASTNode ast) throws SemanticException { switch (ast.getChildCount()) { case 0: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index f5d0602..b1a2b24 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -146,6 +146,7 @@ TOK_ALTERTABLE_ARCHIVE; TOK_ALTERTABLE_UNARCHIVE; TOK_ALTERTABLE_SERDEPROPERTIES; TOK_ALTERTABLE_SERIALIZER; +TOK_ALTERTABLE_UPDATECOLSTATS; TOK_TABLE_PARTITION; TOK_ALTERTABLE_FILEFORMAT; TOK_ALTERTABLE_LOCATION; @@ -937,6 +938,7 @@ alterTableStatementSuffix : alterStatementSuffixRename | alterStatementSuffixAddCol | alterStatementSuffixRenameCol + | alterStatementSuffixUpdateStatsCol | alterStatementSuffixDropPartitions | alterStatementSuffixAddPartitions | alterStatementSuffixTouch @@ -1029,6 +1031,13 @@ alterStatementSuffixRenameCol ->^(TOK_ALTERTABLE_RENAMECOL identifier $oldName $newName colType $comment? alterStatementChangeColPosition?) ; +alterStatementSuffixUpdateStatsCol +@init { pushMsg("update column statistics", state); } +@after { popMsg(state); } + : identifier KW_UPDATE KW_STATISTICS KW_FOR KW_COLUMN? colName=identifier KW_SET tableProperties (KW_COMMENT comment=StringLiteral)? + ->^(TOK_ALTERTABLE_UPDATECOLSTATS identifier $colName tableProperties $comment?) + ; + alterStatementChangeColPosition : first=KW_FIRST|KW_AFTER afterCol=identifier ->{$first != null}? ^(TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION ) @@ -1131,6 +1140,7 @@ alterTblPartitionStatementSuffix | alterStatementSuffixMergeFiles | alterStatementSuffixSerdeProperties | alterStatementSuffixRenamePart + | alterStatementSuffixStatsPart | alterStatementSuffixBucketNum | alterTblPartitionStatementSuffixSkewedLocation | alterStatementSuffixClusterbySortby @@ -1222,6 +1232,13 @@ alterStatementSuffixRenamePart ->^(TOK_ALTERTABLE_RENAMEPART partitionSpec) ; +alterStatementSuffixStatsPart +@init { pushMsg("alter table stats partition statement", state); } +@after { popMsg(state); } + : KW_UPDATE KW_STATISTICS KW_FOR KW_COLUMN? colName=identifier KW_SET tableProperties (KW_COMMENT comment=StringLiteral)? + ->^(TOK_ALTERTABLE_UPDATECOLSTATS $colName tableProperties $comment?) + ; + alterStatementSuffixMergeFiles @init { pushMsg("", state); } @after { popMsg(state); } @@ -1301,6 +1318,7 @@ descStatement | (KW_DESCRIBE|KW_DESC) KW_DATABASE KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?) ; + analyzeStatement @init { pushMsg("analyze statement", state); } @after { popMsg(state); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d38270c..5335bca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -912,6 +912,7 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1) boolean phase1Result = true; QBParseInfo qbp = qb.getParseInfo(); + boolean skipRecursion = false; if (ast.getToken() != null) { @@ -8325,6 +8326,7 @@ private Operator genBodyPlan(QB qb, Operator input, Map aliasT // Go over all the destination tables for (String dest : commonGroupByDestGroup) { + curr = inputs.get(dest); if (qbp.getWhrForClause(dest) != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java index 268920a..3251b72 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java @@ -231,9 +231,12 @@ public static BaseSemanticAnalyzer get(HiveConf conf, ASTNode tree) case HiveParser.TOK_TRUNCATETABLE: case HiveParser.TOK_EXCHANGEPARTITION: case HiveParser.TOK_SHOW_SET_ROLE: - + case HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS: return new DDLSemanticAnalyzer(conf); case HiveParser.TOK_ALTERTABLE_PARTITION: + if (tree.getChildCount() == 2 + && ((ASTNode) tree.getChild(1)).getToken().getType() == HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS) + return new DDLSemanticAnalyzer(conf); HiveOperation commandType = null; Integer type = ((ASTNode) tree.getChild(1)).getToken().getType(); if (tree.getChild(0).getChildCount() > 1) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsUpdateWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsUpdateWork.java new file mode 100644 index 0000000..01349f5 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsUpdateWork.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; +import java.util.HashMap; + +import org.apache.hadoop.hive.ql.exec.ListSinkOperator; + +/** + * Column Stats Update Work. + * + */ +@Explain(displayName = "Column Stats Update Work") +public class ColumnStatsUpdateWork implements Serializable { + private static final long serialVersionUID = 1L; + private ColumnStatsDesc colStats; + private String partName; + private HashMap mapProp; + private static final int LIMIT = -1; + + public ColumnStatsUpdateWork() { + } + + public ColumnStatsUpdateWork(ColumnStatsDesc colStats, String partName, + HashMap mapProp) { + this.setPartName(partName); + this.setColStats(colStats); + this.mapProp = mapProp; + } + + @Override + public String toString() { + return null; + } + + @Explain(displayName = "Column Stats Desc") + public ColumnStatsDesc getColStats() { + return colStats; + } + + + public String getPartName() { + return partName; + } + + public void setPartName(String partName) { + this.partName = partName; + } + + public void setColStats(ColumnStatsDesc colStats) { + this.colStats = colStats; + } + + public HashMap getMapProp() { + return mapProp; + } + + public void setMapProp(HashMap mapProp) { + this.mapProp = mapProp; + } + + public static int getLimit() { + return LIMIT; + } + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java index 79d9d16..99e863c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java @@ -864,6 +864,7 @@ public void setShowIndexesDesc(ShowIndexesDesc showIndexesDesc) { */ @Explain(displayName = "Describe Table Operator") public DescTableDesc getDescTblDesc() { + return descTblDesc; }