diff --git hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java index 18bf172..05a0c22 100644 --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java @@ -237,7 +237,7 @@ public void postAnalyze(HiveSemanticAnalyzerHookContext context, case HiveParser.TOK_ALTERTABLE_EXCHANGEPARTITION: case HiveParser.TOK_ALTERTABLE_SKEWED: case HiveParser.TOK_ALTERTABLE_FILEFORMAT: - case HiveParser.TOK_ALTERTABLE_LOCATION: + case HiveParser.TOK_LOCATION: case HiveParser.TOK_ALTERTABLE_MERGEFILES: case HiveParser.TOK_ALTERTABLE_RENAMEPART: case HiveParser.TOK_ALTERTABLE_SKEWED_LOCATION: diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 3f58130..869d356 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -165,7 +165,7 @@ import java.util.Map.Entry; import java.util.Set; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DATABASELOCATION; +import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_LOCATION; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DATABASEPROPERTIES; /** @@ -298,7 +298,7 @@ public void analyzeInternal(ASTNode input) throws SemanticException { analyzeExchangePartition(qualified, ast); } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_FILEFORMAT) { analyzeAlterTableFileFormat(ast, tableName, partSpec); - } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_LOCATION) { + } else if (ast.getToken().getType() == HiveParser.TOK_LOCATION) { analyzeAlterTableLocation(ast, tableName, partSpec); } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_MERGEFILES) { analyzeAlterTablePartMergeFiles(ast, tableName, partSpec); @@ -803,7 +803,7 @@ private void analyzeCreateDatabase(ASTNode ast) throws SemanticException { case TOK_DATABASEPROPERTIES: dbProps = DDLSemanticAnalyzer.getProps((ASTNode) childNode.getChild(0)); break; - case TOK_DATABASELOCATION: + case TOK_LOCATION: dbLocation = unescapeSQLString(childNode.getChild(0).getText()); addLocationToOutputs(dbLocation); break; @@ -1137,7 +1137,7 @@ private void analyzeCreateIndex(ASTNode ast) throws SemanticException { case HiveParser.TOK_DEFERRED_REBUILDINDEX: deferredRebuild = true; break; - case HiveParser.TOK_TABLELOCATION: + case HiveParser.TOK_LOCATION: location = unescapeSQLString(child.getChild(0).getText()); addLocationToOutputs(location); break; @@ -2809,6 +2809,20 @@ private void analyzeAlterTablePartColType(String[] qualified, ASTNode ast) alterTblAlterPartDesc), conf)); } + private void addPartitionToPartitionDesc(Map part, StorageFormat storageFormat, + Map serdeProperties, String location, AddPartitionDesc addPartitionDesc) { + addPartitionDesc.addPartition(part, location); + + OnePartitionDesc addedPart = + addPartitionDesc.getPartition(addPartitionDesc.getPartitionCount() - 1); + addedPart.setSerdeParams(serdeProperties); + if (storageFormat != null) { + addedPart.setSerializationLib(storageFormat.getSerde()); + addedPart.setInputFormat(storageFormat.getInputFormat()); + addedPart.setOutputFormat(storageFormat.getOutputFormat()); + } + } + /** * Add one or more partitions to a table. Useful when the data has been copied * to the right location by some other process. @@ -2836,6 +2850,8 @@ private void analyzeAlterTableAddParts(String[] qualified, CommonTree ast, boole int numCh = ast.getChildCount(); int start = ifNotExists ? 1 : 0; + StorageFormat currentStorageFormat = null; + Map currentSerdeProperties = null; String currentLocation = null; Map currentPart = null; // Parser has done some verification, so the order of tokens doesn't need to be verified here. @@ -2846,13 +2862,30 @@ private void analyzeAlterTableAddParts(String[] qualified, CommonTree ast, boole switch (child.getToken().getType()) { case HiveParser.TOK_PARTSPEC: if (currentPart != null) { - addPartitionDesc.addPartition(currentPart, currentLocation); + addPartitionToPartitionDesc(currentPart, currentStorageFormat, currentSerdeProperties, currentLocation, + addPartitionDesc); + + currentStorageFormat = null; + currentSerdeProperties = null; currentLocation = null; } currentPart = getValidatedPartSpec(tab, child, conf, true); validatePartitionValues(currentPart); // validate reserved values break; - case HiveParser.TOK_PARTITIONLOCATION: + case HiveParser.TOK_PARTITIONFILEFORMAT: + if (isView) { + throw new SemanticException("FILEFORMAT clause illegal for view partition"); + } + currentStorageFormat = new StorageFormat(conf); + currentStorageFormat.fillStorageFormat((ASTNode) child.getChild(0)); + break; + case HiveParser.TOK_PARTITIONSERDEPROPERTIES: + if (isView) { + throw new SemanticException("SERDEPROPERTIES clause illegal for view partition"); + } + currentSerdeProperties = getProps((ASTNode) child.getChild(0).getChild(0)); + break; + case HiveParser.TOK_LOCATION: // if location specified, set in partition if (isView) { throw new SemanticException("LOCATION clause illegal for view partition"); @@ -2867,7 +2900,8 @@ private void analyzeAlterTableAddParts(String[] qualified, CommonTree ast, boole // add the last one if (currentPart != null) { - addPartitionDesc.addPartition(currentPart, currentLocation); + addPartitionToPartitionDesc(currentPart, currentStorageFormat, currentSerdeProperties, currentLocation, + addPartitionDesc); } if (this.conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 55915a6..18bd3e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -173,7 +173,6 @@ TOK_ALTERTABLE_UPDATECOLSTATS; TOK_ALTERTABLE_UPDATESTATS; TOK_TABLE_PARTITION; TOK_ALTERTABLE_FILEFORMAT; -TOK_ALTERTABLE_LOCATION; TOK_ALTERTABLE_PROPERTIES; TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION; TOK_ALTERTABLE_DROPPROPERTIES; @@ -219,6 +218,7 @@ TOK_TABLEROWFORMATLINES; TOK_TABLEROWFORMATNULL; TOK_TABLEFILEFORMAT; TOK_FILEFORMAT_GENERIC; +TOK_LOCATION; TOK_OFFLINE; TOK_ENABLE; TOK_DISABLE; @@ -228,8 +228,8 @@ TOK_STORAGEHANDLER; TOK_NOT_CLUSTERED; TOK_NOT_SORTED; TOK_TABCOLNAME; -TOK_TABLELOCATION; -TOK_PARTITIONLOCATION; +TOK_PARTITIONFILEFORMAT; +TOK_PARTITIONSERDEPROPERTIES; TOK_TABLEBUCKETSAMPLE; TOK_TABLESPLITSAMPLE; TOK_PERCENT; @@ -325,7 +325,6 @@ TOK_SHOWDBLOCKS; TOK_INDEXCOMMENT; TOK_DESCDATABASE; TOK_DATABASEPROPERTIES; -TOK_DATABASELOCATION; TOK_DBPROPLIST; TOK_ALTERDATABASE_PROPERTIES; TOK_ALTERDATABASE_OWNER; @@ -782,8 +781,8 @@ importStatement : KW_IMPORT ((ext=KW_EXTERNAL)? KW_TABLE (tab=tableOrPartition))? KW_FROM (path=StringLiteral) - tableLocation? - -> ^(TOK_IMPORT $path $tab? $ext? tableLocation?) + location? + -> ^(TOK_IMPORT $path $tab? $ext? location?) ; replDumpStatement @@ -903,16 +902,16 @@ createDatabaseStatement ifNotExists? name=identifier databaseComment? - dbLocation? + location? (KW_WITH KW_DBPROPERTIES dbprops=dbProperties)? - -> ^(TOK_CREATEDATABASE $name ifNotExists? dbLocation? databaseComment? $dbprops?) + -> ^(TOK_CREATEDATABASE $name ifNotExists? location? databaseComment? $dbprops?) ; -dbLocation -@init { pushMsg("database location specification", state); } +location +@init { pushMsg("location specification", state); } @after { popMsg(state); } : - KW_LOCATION locn=StringLiteral -> ^(TOK_DATABASELOCATION $locn) + KW_LOCATION locn=StringLiteral -> ^(TOK_LOCATION $locn) ; dbProperties @@ -958,7 +957,7 @@ createTableStatement ( like=KW_LIKE likeName=tableName tableRowFormat? tableFileFormat? - tableLocation? + location? tablePropertiesPrefixed? | (LPAREN columnNameTypeOrPKOrFKList RPAREN)? tableComment? @@ -967,7 +966,7 @@ createTableStatement tableSkewed? tableRowFormat? tableFileFormat? - tableLocation? + location? tablePropertiesPrefixed? (KW_AS selectStatementWithCTE)? ) @@ -980,7 +979,7 @@ createTableStatement tableSkewed? tableRowFormat? tableFileFormat? - tableLocation? + location? tablePropertiesPrefixed? selectStatementWithCTE? ) @@ -1002,7 +1001,7 @@ createIndexStatement indexTblName? tableRowFormat? tableFileFormat? - tableLocation? + location? tablePropertiesPrefixed? indexComment? ->^(TOK_CREATEINDEX $indexName $typeName $tab $indexedCols @@ -1011,7 +1010,7 @@ createIndexStatement indexTblName? tableRowFormat? tableFileFormat? - tableLocation? + location? tablePropertiesPrefixed? indexComment?) ; @@ -1236,7 +1235,7 @@ alterStatementSuffixAddPartitions[boolean table] ; alterStatementSuffixAddPartitionsElement - : partitionSpec partitionLocation? + : partitionSpec partitionFileFormat? partitionSerdeProperties? location? ; alterStatementSuffixTouch @@ -1260,11 +1259,16 @@ alterStatementSuffixUnArchive -> ^(TOK_ALTERTABLE_UNARCHIVE (partitionSpec)*) ; -partitionLocation -@init { pushMsg("partition location", state); } +partitionFileFormat +@init { pushMsg("partition fileformat", state); } @after { popMsg(state); } - : - KW_LOCATION locn=StringLiteral -> ^(TOK_PARTITIONLOCATION $locn) + : KW_FILEFORMAT fileFormat -> ^(TOK_PARTITIONFILEFORMAT fileFormat) + ; + +partitionSerdeProperties +@init { pushMsg("partition serdeproperties", state); } +@after { popMsg(state); } + : KW_SERDEPROPERTIES tableProperties -> ^(TOK_PARTITIONSERDEPROPERTIES tableProperties) ; alterStatementSuffixDropPartitions[boolean table] @@ -1355,8 +1359,7 @@ skewedLocationMap alterStatementSuffixLocation @init {pushMsg("alter location", state);} @after {popMsg(state);} - : KW_SET KW_LOCATION newLoc=StringLiteral - -> ^(TOK_ALTERTABLE_LOCATION $newLoc) + : KW_SET location -> location ; @@ -1832,14 +1835,14 @@ createMaterializedViewStatement } @after { popMsg(state); } : KW_CREATE KW_MATERIALIZED KW_VIEW (ifNotExists)? name=tableName - tableComment? tableRowFormat? tableFileFormat? tableLocation? + tableComment? tableRowFormat? tableFileFormat? location? tablePropertiesPrefixed? KW_AS selectStatementWithCTE -> ^(TOK_CREATE_MATERIALIZED_VIEW $name ifNotExists? tableComment? tableRowFormat? tableFileFormat? - tableLocation? + location? tablePropertiesPrefixed? selectStatementWithCTE ) @@ -2044,13 +2047,6 @@ tableFileFormat -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; -tableLocation -@init { pushMsg("table location specification", state); } -@after { popMsg(state); } - : - KW_LOCATION locn=StringLiteral -> ^(TOK_TABLELOCATION $locn) - ; - columnNameTypeList @init { pushMsg("column name type list", state); } @after { popMsg(state); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index ce952c5..7d6e9f4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -112,7 +112,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { case HiveParser.KW_EXTERNAL: isExternalSet = true; break; - case HiveParser.TOK_TABLELOCATION: + case HiveParser.TOK_LOCATION: isLocationSet = true; parsedLocation = EximUtil.relativeToAbsolutePath(conf, unescapeSQLString(child.getChild(0).getText())); break; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 79e55b2..bd1b28d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -11687,7 +11687,7 @@ ASTNode analyzeCreateTable( case HiveParser.TOK_TABLEROWFORMAT: rowFormatParams.analyzeRowFormat(child); break; - case HiveParser.TOK_TABLELOCATION: + case HiveParser.TOK_LOCATION: location = unescapeSQLString(child.getChild(0).getText()); location = EximUtil.relativeToAbsolutePath(conf, location); inputs.add(toReadEntity(location)); @@ -11958,7 +11958,7 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt case HiveParser.TOK_TABLEROWFORMAT: rowFormatParams.analyzeRowFormat(child); break; - case HiveParser.TOK_TABLELOCATION: + case HiveParser.TOK_LOCATION: location = unescapeSQLString(child.getChild(0).getText()); location = EximUtil.relativeToAbsolutePath(conf, location); inputs.add(toReadEntity(location)); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java index 520d3de..a65391d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java @@ -138,7 +138,7 @@ tablePartitionCommandType.put(HiveParser.TOK_ALTERTABLE_FILEFORMAT, new HiveOperation[] { HiveOperation.ALTERTABLE_FILEFORMAT, HiveOperation.ALTERPARTITION_FILEFORMAT }); - tablePartitionCommandType.put(HiveParser.TOK_ALTERTABLE_LOCATION, + tablePartitionCommandType.put(HiveParser.TOK_LOCATION, new HiveOperation[] { HiveOperation.ALTERTABLE_LOCATION, HiveOperation.ALTERPARTITION_LOCATION }); tablePartitionCommandType.put(HiveParser.TOK_ALTERTABLE_MERGEFILES, diff --git ql/src/test/queries/clientpositive/add_part_fileformat_serdeproperties_location.q ql/src/test/queries/clientpositive/add_part_fileformat_serdeproperties_location.q new file mode 100644 index 0000000..2659357 --- /dev/null +++ ql/src/test/queries/clientpositive/add_part_fileformat_serdeproperties_location.q @@ -0,0 +1,40 @@ +CREATE TABLE add_part_test (a STRING) PARTITIONED BY (b INT); + +-- Test different combinations and orderings of FILEFORMAT, SERDEPROPERTIES, LOCATION +EXPLAIN ALTER TABLE add_part_test ADD IF NOT EXISTS +PARTITION (b=1) +PARTITION (b=2) FILEFORMAT ORC +PARTITION (b=3) SERDEPROPERTIES ('foo' = 'bar') +PARTITION (b=4) LOCATION 'A' +PARTITION (b=5) FILEFORMAT + INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' + SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' + SERDEPROPERTIES ('hello' = 'world') +PARTITION (b=6) FILEFORMAT SEQUENCEFILE LOCATION 'B' +PARTITION (b=7) SERDEPROPERTIES ('a' = 'b', 'c' = 'd') LOCATION 'C' +PARTITION (b=8) FILEFORMAT PARQUET SERDEPROPERTIES ('e' = 'f') LOCATION 'D'; + +ALTER TABLE add_part_test ADD IF NOT EXISTS +PARTITION (b=1) +PARTITION (b=2) FILEFORMAT ORC +PARTITION (b=3) SERDEPROPERTIES ('foo' = 'bar') +PARTITION (b=4) LOCATION 'A' +PARTITION (b=5) FILEFORMAT + INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' + SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' + SERDEPROPERTIES ('hello' = 'world') +PARTITION (b=6) FILEFORMAT SEQUENCEFILE LOCATION 'B' +PARTITION (b=7) SERDEPROPERTIES ('a' = 'b', 'c' = 'd') LOCATION 'C' +PARTITION (b=8) FILEFORMAT PARQUET SERDEPROPERTIES ('e' = 'f') LOCATION 'D'; + +DESCRIBE FORMATTED add_part_test; +DESCRIBE FORMATTED add_part_test PARTITION (b=1); +DESCRIBE FORMATTED add_part_test PARTITION (b=2); +DESCRIBE FORMATTED add_part_test PARTITION (b=3); +DESCRIBE FORMATTED add_part_test PARTITION (b=4); +DESCRIBE FORMATTED add_part_test PARTITION (b=5); +DESCRIBE FORMATTED add_part_test PARTITION (b=6); +DESCRIBE FORMATTED add_part_test PARTITION (b=7); +DESCRIBE FORMATTED add_part_test PARTITION (b=8); diff --git ql/src/test/results/clientpositive/add_part_fileformat_serdeproperties_location.q.out ql/src/test/results/clientpositive/add_part_fileformat_serdeproperties_location.q.out new file mode 100644 index 0000000..d75a635 --- /dev/null +++ ql/src/test/results/clientpositive/add_part_fileformat_serdeproperties_location.q.out @@ -0,0 +1,398 @@ +PREHOOK: query: CREATE TABLE add_part_test (a STRING) PARTITIONED BY (b INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@add_part_test +POSTHOOK: query: CREATE TABLE add_part_test (a STRING) PARTITIONED BY (b INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@add_part_test +PREHOOK: query: -- Test different combinations and orderings of FILEFORMAT, SERDEPROPERTIES, LOCATION +EXPLAIN ALTER TABLE add_part_test ADD IF NOT EXISTS +PARTITION (b=1) +PARTITION (b=2) FILEFORMAT ORC +PARTITION (b=3) SERDEPROPERTIES ('foo' = 'bar') +#### A masked pattern was here #### +PARTITION (b=5) FILEFORMAT + INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' + SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' + SERDEPROPERTIES ('hello' = 'world') +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: query: -- Test different combinations and orderings of FILEFORMAT, SERDEPROPERTIES, LOCATION +EXPLAIN ALTER TABLE add_part_test ADD IF NOT EXISTS +PARTITION (b=1) +PARTITION (b=2) FILEFORMAT ORC +PARTITION (b=3) SERDEPROPERTIES ('foo' = 'bar') +#### A masked pattern was here #### +PARTITION (b=5) FILEFORMAT + INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' + SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' + SERDEPROPERTIES ('hello' = 'world') +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Add Partition Operator: +#### A masked pattern was here #### + Spec: {b=1}, {b=2}, {b=3}, {b=4}, {b=5}, {b=6}, {b=7}, {b=8} + +PREHOOK: query: ALTER TABLE add_part_test ADD IF NOT EXISTS +PARTITION (b=1) +PARTITION (b=2) FILEFORMAT ORC +PARTITION (b=3) SERDEPROPERTIES ('foo' = 'bar') +#### A masked pattern was here #### +PARTITION (b=5) FILEFORMAT + INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' + SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' + SERDEPROPERTIES ('hello' = 'world') +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@add_part_test +POSTHOOK: query: ALTER TABLE add_part_test ADD IF NOT EXISTS +PARTITION (b=1) +PARTITION (b=2) FILEFORMAT ORC +PARTITION (b=3) SERDEPROPERTIES ('foo' = 'bar') +#### A masked pattern was here #### +PARTITION (b=5) FILEFORMAT + INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' + SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' + SERDEPROPERTIES ('hello' = 'world') +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@add_part_test +POSTHOOK: Output: default@add_part_test@b=1 +POSTHOOK: Output: default@add_part_test@b=2 +POSTHOOK: Output: default@add_part_test@b=3 +POSTHOOK: Output: default@add_part_test@b=4 +POSTHOOK: Output: default@add_part_test@b=5 +POSTHOOK: Output: default@add_part_test@b=6 +POSTHOOK: Output: default@add_part_test@b=7 +POSTHOOK: Output: default@add_part_test@b=8 +PREHOOK: query: DESCRIBE FORMATTED add_part_test +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@add_part_test +POSTHOOK: query: DESCRIBE FORMATTED add_part_test +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@add_part_test +# col_name data_type comment + +a string + +# Partition Information +# col_name data_type comment + +b int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@add_part_test +POSTHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@add_part_test +# col_name data_type comment + +a string + +# Partition Information +# col_name data_type comment + +b int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: add_part_test +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=2) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@add_part_test +POSTHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=2) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@add_part_test +# col_name data_type comment + +a string + +# Partition Information +# col_name data_type comment + +b int + +# Detailed Partition Information +Partition Value: [2] +Database: default +Table: add_part_test +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=3) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@add_part_test +POSTHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=3) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@add_part_test +# col_name data_type comment + +a string + +# Partition Information +# col_name data_type comment + +b int + +# Detailed Partition Information +Partition Value: [3] +Database: default +Table: add_part_test +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + foo bar +PREHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=4) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@add_part_test +POSTHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=4) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@add_part_test +# col_name data_type comment + +a string + +# Partition Information +# col_name data_type comment + +b int + +# Detailed Partition Information +Partition Value: [4] +Database: default +Table: add_part_test +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=5) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@add_part_test +POSTHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=5) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@add_part_test +# col_name data_type comment + +a string + +# Partition Information +# col_name data_type comment + +b int + +# Detailed Partition Information +Partition Value: [5] +Database: default +Table: add_part_test +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.avro.AvroSerDe +InputFormat: org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + hello world +PREHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=6) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@add_part_test +POSTHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=6) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@add_part_test +# col_name data_type comment + +a string + +# Partition Information +# col_name data_type comment + +b int + +# Detailed Partition Information +Partition Value: [6] +Database: default +Table: add_part_test +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat +OutputFormat: org.apache.hadoop.mapred.SequenceFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=7) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@add_part_test +POSTHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=7) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@add_part_test +# col_name data_type comment + +a string + +# Partition Information +# col_name data_type comment + +b int + +# Detailed Partition Information +Partition Value: [7] +Database: default +Table: add_part_test +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + a b + c d +PREHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=8) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@add_part_test +POSTHOOK: query: DESCRIBE FORMATTED add_part_test PARTITION (b=8) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@add_part_test +# col_name data_type comment + +a string + +# Partition Information +# col_name data_type comment + +b int + +# Detailed Partition Information +Partition Value: [8] +Database: default +Table: add_part_test +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + e f