diff --git data/files/grad2.avsc data/files/grad2.avsc new file mode 100644 index 0000000000000000000000000000000000000000..44f797ff49e51548e4afad9c862b6470f7b78ef4 --- /dev/null +++ data/files/grad2.avsc @@ -0,0 +1,9 @@ +{"namespace": "devmode", +"type": "record", +"name": "gradinfo", +"fields": [ +{"name": "colX", "type": "string"}, +{"name": "colY", "type": "string"}, +{"name": "colZ", "type": "double"} +] +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 4eafcde7fb397a1dbf731aa4a9c48f835688ec27..a1edee9a054a8580ea6830de6d4cfeb7467ee621 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -111,6 +111,7 @@ import org.apache.hadoop.hive.metastore.api.WMResourcePlanStatus; import org.apache.hadoop.hive.metastore.api.WMTrigger; import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -4215,6 +4216,8 @@ private static StorageDescriptor retrieveStorageDescriptor(Table tbl, Partition } tbl.setNumBuckets(alterTbl.getNumberBuckets()); } + } else if (alterTbl.getOp() == AlterTableTypes.UPDATECOLUMNS) { + updateColumns(tbl, part); } else { throw new HiveException(ErrorMsg.UNSUPPORTED_ALTER_TBL_OP, alterTbl.getOp().toString()); } @@ -4438,6 +4441,29 @@ private int addConstraints(Hive db, AlterTableDesc alterTbl) return 0; } + private int updateColumns(Table tbl, Partition part) + throws HiveException { + String serializationLib = tbl.getSd().getSerdeInfo().getSerializationLib(); + if (MetastoreConf.getStringCollection(conf, + MetastoreConf.ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA).contains(serializationLib)) { + throw new HiveException(tbl.getTableName() + " has serde " + serializationLib + " for which schema " + + "is already handled by HMS."); + } + Deserializer deserializer = tbl.getDeserializer(true); + try { + LOG.info("Updating metastore columns for table: {}", tbl.getTableName()); + final List fields = HiveMetaStoreUtils.getFieldsFromDeserializer( + tbl.getTableName(), deserializer); + StorageDescriptor sd = retrieveStorageDescriptor(tbl, part); + sd.setCols(fields); + } catch (org.apache.hadoop.hive.serde2.SerDeException | MetaException e) { + LOG.error("alter table update columns: {}", e); + throw new HiveException(e, ErrorMsg.GENERIC_ERROR); + } + + return 0; + } + /** * Drop a given table or some partitions. DropTableDesc is currently used for both. * diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 1605d7dd3f15b2e4186dd8fd8f5bd086ba065c8e..7c159524ab844fcca6d6c695c30d2f5b370d72ef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -354,6 +354,8 @@ public void analyzeInternal(ASTNode input) throws SemanticException { analyzeAlterTableDropConstraint(ast, tableName); } else if(ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_ADDCONSTRAINT) { analyzeAlterTableAddConstraint(ast, tableName); + } else if(ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_UPDATECOLUMNS) { + analyzeAlterTableUpdateColumns(ast, tableName, partSpec); } break; } @@ -2177,6 +2179,23 @@ private void analyzeAlterTableAddConstraint(ASTNode ast, String tableName) alterTblDesc))); } + private void analyzeAlterTableUpdateColumns(ASTNode ast, String tableName, + HashMap partSpec) throws SemanticException { + + boolean isCascade = false; + if (null != ast.getFirstChildWithType(HiveParser.TOK_CASCADE)) { + isCascade = true; + } + + AlterTableDesc alterTblDesc = new AlterTableDesc(AlterTableTypes.UPDATECOLUMNS); + alterTblDesc.setOldName(tableName); + alterTblDesc.setIsCascade(isCascade); + alterTblDesc.setPartSpec(partSpec); + + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), + alterTblDesc), conf)); + } + static HashMap getProps(ASTNode prop) { // Must be deterministic order map for consistent q-test output across Java versions HashMap mapProp = new LinkedHashMap(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 3abc75248b07b63fb8055b50d1eb7f41a407611e..56d7ea22e0a65fef4c4b927f377c581c53d9edd5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -186,6 +186,7 @@ TOK_ALTERTABLE_CLUSTER_SORT; TOK_ALTERTABLE_COMPACT; TOK_ALTERTABLE_DROPCONSTRAINT; TOK_ALTERTABLE_ADDCONSTRAINT; +TOK_ALTERTABLE_UPDATECOLUMNS; TOK_MSCK; TOK_SHOWDATABASES; TOK_SHOWTABLES; @@ -1149,6 +1150,7 @@ alterTblPartitionStatementSuffix | alterStatementSuffixUpdateStats | alterStatementSuffixRenameCol | alterStatementSuffixAddCol + | alterStatementSuffixUpdateColumns ; alterStatementPartitionKeyType @@ -1228,6 +1230,13 @@ alterStatementSuffixAddConstraint -> ^(TOK_ALTERTABLE_ADDCONSTRAINT alterConstraintWithName) ; +alterStatementSuffixUpdateColumns +@init { pushMsg("update columns statement", state); } +@after { popMsg(state); } + : KW_UPDATE KW_COLUMNS restrictOrCascade? + -> ^(TOK_ALTERTABLE_UPDATECOLUMNS restrictOrCascade?) + ; + alterStatementSuffixDropConstraint @init { pushMsg("drop constraint statement", state); } @after { popMsg(state); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java index 78f83ef03936ff31dafcb612e821c5580ef7af82..f46a7b5de0115332ad504b513eeeab657bf95d55 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java @@ -66,6 +66,7 @@ HiveOperation.ALTERTABLE_EXCHANGEPARTITION); commandType.put(HiveParser.TOK_ALTERTABLE_DROPCONSTRAINT, HiveOperation.ALTERTABLE_DROPCONSTRAINT); commandType.put(HiveParser.TOK_ALTERTABLE_ADDCONSTRAINT, HiveOperation.ALTERTABLE_ADDCONSTRAINT); + commandType.put(HiveParser.TOK_ALTERTABLE_UPDATECOLUMNS, HiveOperation.ALTERTABLE_UPDATECOLUMNS); commandType.put(HiveParser.TOK_SHOWDATABASES, HiveOperation.SHOWDATABASES); commandType.put(HiveParser.TOK_SHOWTABLES, HiveOperation.SHOWTABLES); commandType.put(HiveParser.TOK_SHOWCOLUMNS, HiveOperation.SHOWCOLUMNS); @@ -242,6 +243,7 @@ private static BaseSemanticAnalyzer getInternal(QueryState queryState, ASTNode t case HiveParser.TOK_ALTERTABLE_SKEWED: case HiveParser.TOK_ALTERTABLE_DROPCONSTRAINT: case HiveParser.TOK_ALTERTABLE_ADDCONSTRAINT: + case HiveParser.TOK_ALTERTABLE_UPDATECOLUMNS: queryState.setCommandType(commandType.get(child.getType())); return new DDLSemanticAnalyzer(queryState); } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java index bf86cec488f6296cc020ce0f0b1430605362ffd9..479eda2f7bc78e3dd1eb2ee44c201c849496c5f2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java @@ -64,7 +64,8 @@ DROPPARTITION("drop partition"), RENAMEPARTITION("rename partition"), ADDSKEWEDBY("add skew column"), ALTERSKEWEDLOCATION("alter skew location"), ALTERBUCKETNUM("alter bucket number"), ALTERPARTITION("alter partition"), COMPACT("compact"), - TRUNCATE("truncate"), MERGEFILES("merge files"), DROPCONSTRAINT("drop constraint"), ADDCONSTRAINT("add constraint"); + TRUNCATE("truncate"), MERGEFILES("merge files"), DROPCONSTRAINT("drop constraint"), ADDCONSTRAINT("add constraint"), + UPDATECOLUMNS("update columns"); ; private final String name; @@ -894,6 +895,13 @@ public boolean getIsCascade() { return isCascade; } + /** + * @param cascade the isCascade to set + */ + public void setIsCascade(boolean isCascade) { + this.isCascade = isCascade; + } + public static boolean doesAlterTableTypeSupportPartialPartitionSpec(AlterTableTypes type) { return alterTableTypesWithPartialSpec.contains(type); } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java index a9e5c8cffeaf3ac764a78a711cca6c9f67e33ee2..cd4c206a89f1bc1a6195b0f1f39d3c4b462dc027 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java @@ -127,6 +127,8 @@ new Privilege[]{Privilege.ALTER_METADATA}, null), ALTERTABLE_ADDCONSTRAINT("ALTERTABLE_ADDCONSTRAINT", new Privilege[]{Privilege.ALTER_METADATA}, null), + ALTERTABLE_UPDATECOLUMNS("ALTERTABLE_UPDATECOLUMNS", + new Privilege[]{Privilege.ALTER_METADATA}, null), ALTERVIEW_RENAME("ALTERVIEW_RENAME", new Privilege[] {Privilege.ALTER_METADATA}, null), ALTERVIEW_AS("ALTERVIEW_AS", new Privilege[] {Privilege.ALTER_METADATA}, null), ALTERTABLE_COMPACT("ALTERTABLE_COMPACT", new Privilege[]{Privilege.SELECT}, new Privilege[]{Privilege.ALTER_DATA}), diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java index 5d6905a4e650a01eaeb10d85759ab1ced01d9b1b..60599f3206bf6a35eec0cf1f826740b93f09f4d1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java @@ -130,6 +130,7 @@ ALTERVIEW_AS, ALTER_MATERIALIZED_VIEW_REWRITE, ALTERTABLE_COMPACT, + ALTERTABLE_UPDATECOLUMNS, SHOW_COMPACTIONS, SHOW_TRANSACTIONS, ABORT_TRANSACTIONS, diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java index a55e66b4b67bad99996495f88abffc4366be3df8..e408ea4175b9b974d79d45d0ac132da932b608fd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java @@ -233,6 +233,8 @@ public HivePrivilegeObjectType getObjectType() { op2Priv.put(HiveOperationType.ALTERTBLPART_SKEWED_LOCATION, PrivRequirement.newIOPrivRequirement (OWNER_PRIV_AR, INS_SEL_DEL_NOGRANT_AR)); op2Priv.put(HiveOperationType.ALTERTABLE_COMPACT, PrivRequirement.newIOPrivRequirement +(OWNER_PRIV_AR, OWNER_PRIV_AR)); + op2Priv.put(HiveOperationType.ALTERTABLE_UPDATECOLUMNS, PrivRequirement.newIOPrivRequirement (OWNER_PRIV_AR, OWNER_PRIV_AR)); op2Priv.put(HiveOperationType.TRUNCATETABLE, PrivRequirement.newIOPrivRequirement (OWNER_PRIV_AR, OWNER_PRIV_AR)); diff --git ql/src/test/queries/clientnegative/hms_using_serde_alter_table_update_columns.q ql/src/test/queries/clientnegative/hms_using_serde_alter_table_update_columns.q new file mode 100644 index 0000000000000000000000000000000000000000..e323d1ca84bf95402dae45d73d3fb29584c545d3 --- /dev/null +++ ql/src/test/queries/clientnegative/hms_using_serde_alter_table_update_columns.q @@ -0,0 +1,6 @@ +-- verify that for HMS using serde's UPDATE COLUMNS returns error + +CREATE TABLE hmsserdetable (name string) STORED AS PARQUET; +DESCRIBE hmsserdetable; + +ALTER TABLE hmsserdetable UPDATE COLUMNS; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/avro_alter_table_update_columns.q ql/src/test/queries/clientpositive/avro_alter_table_update_columns.q new file mode 100644 index 0000000000000000000000000000000000000000..4b03ffff8c9aa5f18701d01795b5421cbc2a29ae --- /dev/null +++ ql/src/test/queries/clientpositive/avro_alter_table_update_columns.q @@ -0,0 +1,116 @@ +-- verify schema changes introduced in avro.schema.literal/url sync with HMS if ALTER TABLE UPDATE COLUMNS is called + +CREATE TABLE avro_extschema_literal + STORED AS AVRO + TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "org.apache.hive", + "name": "ext_schema", + "type": "record", + "fields": [ + { "name":"number", "type":"int" }, + { "name":"first_name", "type":"string" }, + { "name":"last_name", "type":"string" } + ] }'); +DESCRIBE avro_extschema_literal; + +ALTER TABLE avro_extschema_literal SET + TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "org.apache.hive", + "name": "ext_schema", + "type": "record", + "fields": [ + { "name":"newCol", "type":"int" } + ] }'); +DESCRIBE avro_extschema_literal; + +ALTER TABLE avro_extschema_literal UNSET TBLPROPERTIES ('avro.schema.literal'); +DESCRIBE avro_extschema_literal; + +ALTER TABLE avro_extschema_literal SET + TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "org.apache.hive", + "name": "ext_schema", + "type": "record", + "fields": [ + { "name":"newCol", "type":"int" } + ] }'); +ALTER TABLE avro_extschema_literal UPDATE COLUMNS CASCADE; +DESCRIBE avro_extschema_literal; + +ALTER TABLE avro_extschema_literal UNSET TBLPROPERTIES ('avro.schema.literal'); +DESCRIBE avro_extschema_literal; + +dfs -cp ${system:hive.root}data/files/grad.avsc ${system:test.tmp.dir}/; +dfs -cp ${system:hive.root}data/files/grad2.avsc ${system:test.tmp.dir}/; + + +CREATE TABLE avro_extschema_url + STORED AS AVRO + TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/grad.avsc'); +DESCRIBE avro_extschema_url; + +ALTER TABLE avro_extschema_url SET + TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/grad2.avsc'); +DESCRIBE avro_extschema_url; + +ALTER TABLE avro_extschema_url UNSET TBLPROPERTIES ('avro.schema.url'); +DESCRIBE avro_extschema_url; + + +ALTER TABLE avro_extschema_url SET + TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/grad2.avsc'); +ALTER TABLE avro_extschema_url UPDATE COLUMNS CASCADE; +DESCRIBE avro_extschema_url; + +ALTER TABLE avro_extschema_url UNSET TBLPROPERTIES ('avro.schema.url'); +DESCRIBE avro_extschema_url; + + + +--testing partition level and non-cascade options + +CREATE TABLE avro_extschema_url_parted + PARTITIONED BY (p1 string, p2 string) + STORED AS AVRO + TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/grad.avsc'); +ALTER TABLE avro_extschema_url_parted + ADD PARTITION (p1=2017, p2=11); +ALTER TABLE avro_extschema_url_parted + ADD PARTITION (p1=2018, p2=2); +ALTER TABLE avro_extschema_url_parted + ADD PARTITION (p1=2018, p2=3); + +DESCRIBE avro_extschema_url_parted; + +--case: partial partition spec +ALTER TABLE avro_extschema_url_parted SET + TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/grad2.avsc'); +ALTER TABLE avro_extschema_url_parted PARTITION (p1=2018) UPDATE COLUMNS; +ALTER TABLE avro_extschema_url_parted UNSET TBLPROPERTIES ('avro.schema.url'); + +DESCRIBE avro_extschema_url_parted; +DESCRIBE avro_extschema_url_parted PARTITION (p1=2017, p2=11); +DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=2); +DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=3); + +--case: table with restrict (no cascade) +ALTER TABLE avro_extschema_url_parted SET + TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/grad2.avsc'); +ALTER TABLE avro_extschema_url_parted UPDATE COLUMNS; +ALTER TABLE avro_extschema_url_parted UNSET TBLPROPERTIES ('avro.schema.url'); + +DESCRIBE avro_extschema_url_parted; +DESCRIBE avro_extschema_url_parted PARTITION (p1=2017, p2=11); +DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=2); +DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=3); + +--case: full partition spec +ALTER TABLE avro_extschema_url_parted SET + TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/grad2.avsc'); +ALTER TABLE avro_extschema_url_parted PARTITION (p1=2017, p2=11) UPDATE COLUMNS; +ALTER TABLE avro_extschema_url_parted UNSET TBLPROPERTIES ('avro.schema.url'); + +DESCRIBE avro_extschema_url_parted; +DESCRIBE avro_extschema_url_parted PARTITION (p1=2017, p2=11); +DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=2); +DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=3); \ No newline at end of file diff --git ql/src/test/results/clientnegative/hms_using_serde_alter_table_update_columns.q.out ql/src/test/results/clientnegative/hms_using_serde_alter_table_update_columns.q.out new file mode 100644 index 0000000000000000000000000000000000000000..7c82270698d2f607ccf51e66c817a17a448b3871 --- /dev/null +++ ql/src/test/results/clientnegative/hms_using_serde_alter_table_update_columns.q.out @@ -0,0 +1,18 @@ +PREHOOK: query: CREATE TABLE hmsserdetable (name string) STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hmsserdetable +POSTHOOK: query: CREATE TABLE hmsserdetable (name string) STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hmsserdetable +PREHOOK: query: DESCRIBE hmsserdetable +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@hmsserdetable +POSTHOOK: query: DESCRIBE hmsserdetable +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@hmsserdetable +name string +PREHOOK: query: ALTER TABLE hmsserdetable UPDATE COLUMNS +PREHOOK: type: ALTERTABLE_UPDATECOLUMNS +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. default.hmsserdetable has serde org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe for which schema is already handled by HMS. diff --git ql/src/test/results/clientpositive/avro_alter_table_update_columns.q.out ql/src/test/results/clientpositive/avro_alter_table_update_columns.q.out new file mode 100644 index 0000000000000000000000000000000000000000..8985f18b0decfdf9f1fde9110c000ff23bffdc1a --- /dev/null +++ ql/src/test/results/clientpositive/avro_alter_table_update_columns.q.out @@ -0,0 +1,576 @@ +PREHOOK: query: CREATE TABLE avro_extschema_literal + STORED AS AVRO + TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "org.apache.hive", + "name": "ext_schema", + "type": "record", + "fields": [ + { "name":"number", "type":"int" }, + { "name":"first_name", "type":"string" }, + { "name":"last_name", "type":"string" } + ] }') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_extschema_literal +POSTHOOK: query: CREATE TABLE avro_extschema_literal + STORED AS AVRO + TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "org.apache.hive", + "name": "ext_schema", + "type": "record", + "fields": [ + { "name":"number", "type":"int" }, + { "name":"first_name", "type":"string" }, + { "name":"last_name", "type":"string" } + ] }') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_extschema_literal +PREHOOK: query: DESCRIBE avro_extschema_literal +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_literal +POSTHOOK: query: DESCRIBE avro_extschema_literal +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_literal +number int +first_name string +last_name string +PREHOOK: query: ALTER TABLE avro_extschema_literal SET + TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "org.apache.hive", + "name": "ext_schema", + "type": "record", + "fields": [ + { "name":"newCol", "type":"int" } + ] }') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_literal +PREHOOK: Output: default@avro_extschema_literal +POSTHOOK: query: ALTER TABLE avro_extschema_literal SET + TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "org.apache.hive", + "name": "ext_schema", + "type": "record", + "fields": [ + { "name":"newCol", "type":"int" } + ] }') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_literal +POSTHOOK: Output: default@avro_extschema_literal +PREHOOK: query: DESCRIBE avro_extschema_literal +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_literal +POSTHOOK: query: DESCRIBE avro_extschema_literal +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_literal +newcol int +PREHOOK: query: ALTER TABLE avro_extschema_literal UNSET TBLPROPERTIES ('avro.schema.literal') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_literal +PREHOOK: Output: default@avro_extschema_literal +POSTHOOK: query: ALTER TABLE avro_extschema_literal UNSET TBLPROPERTIES ('avro.schema.literal') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_literal +POSTHOOK: Output: default@avro_extschema_literal +PREHOOK: query: DESCRIBE avro_extschema_literal +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_literal +POSTHOOK: query: DESCRIBE avro_extschema_literal +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_literal +number int +first_name string +last_name string +PREHOOK: query: ALTER TABLE avro_extschema_literal SET + TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "org.apache.hive", + "name": "ext_schema", + "type": "record", + "fields": [ + { "name":"newCol", "type":"int" } + ] }') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_literal +PREHOOK: Output: default@avro_extschema_literal +POSTHOOK: query: ALTER TABLE avro_extschema_literal SET + TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "org.apache.hive", + "name": "ext_schema", + "type": "record", + "fields": [ + { "name":"newCol", "type":"int" } + ] }') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_literal +POSTHOOK: Output: default@avro_extschema_literal +PREHOOK: query: ALTER TABLE avro_extschema_literal UPDATE COLUMNS CASCADE +PREHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: query: ALTER TABLE avro_extschema_literal UPDATE COLUMNS CASCADE +POSTHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: Input: default@avro_extschema_literal +POSTHOOK: Output: default@avro_extschema_literal +PREHOOK: query: DESCRIBE avro_extschema_literal +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_literal +POSTHOOK: query: DESCRIBE avro_extschema_literal +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_literal +newcol int +PREHOOK: query: ALTER TABLE avro_extschema_literal UNSET TBLPROPERTIES ('avro.schema.literal') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_literal +PREHOOK: Output: default@avro_extschema_literal +POSTHOOK: query: ALTER TABLE avro_extschema_literal UNSET TBLPROPERTIES ('avro.schema.literal') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_literal +POSTHOOK: Output: default@avro_extschema_literal +PREHOOK: query: DESCRIBE avro_extschema_literal +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_literal +POSTHOOK: query: DESCRIBE avro_extschema_literal +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_literal +newcol int +PREHOOK: query: CREATE TABLE avro_extschema_url + STORED AS AVRO +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_extschema_url +POSTHOOK: query: CREATE TABLE avro_extschema_url + STORED AS AVRO +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_extschema_url +PREHOOK: query: DESCRIBE avro_extschema_url +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url +POSTHOOK: query: DESCRIBE avro_extschema_url +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url +col1 string +col2 string +col3 double +col4 string +col5 string +col6 int +PREHOOK: query: ALTER TABLE avro_extschema_url SET +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url +PREHOOK: Output: default@avro_extschema_url +POSTHOOK: query: ALTER TABLE avro_extschema_url SET +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url +POSTHOOK: Output: default@avro_extschema_url +PREHOOK: query: DESCRIBE avro_extschema_url +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url +POSTHOOK: query: DESCRIBE avro_extschema_url +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url +colx string +coly string +colz double +PREHOOK: query: ALTER TABLE avro_extschema_url UNSET TBLPROPERTIES ('avro.schema.url') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url +PREHOOK: Output: default@avro_extschema_url +POSTHOOK: query: ALTER TABLE avro_extschema_url UNSET TBLPROPERTIES ('avro.schema.url') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url +POSTHOOK: Output: default@avro_extschema_url +PREHOOK: query: DESCRIBE avro_extschema_url +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url +POSTHOOK: query: DESCRIBE avro_extschema_url +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url +col1 string +col2 string +col3 double +col4 string +col5 string +col6 int +PREHOOK: query: ALTER TABLE avro_extschema_url SET +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url +PREHOOK: Output: default@avro_extschema_url +POSTHOOK: query: ALTER TABLE avro_extschema_url SET +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url +POSTHOOK: Output: default@avro_extschema_url +PREHOOK: query: ALTER TABLE avro_extschema_url UPDATE COLUMNS CASCADE +PREHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: query: ALTER TABLE avro_extschema_url UPDATE COLUMNS CASCADE +POSTHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: Input: default@avro_extschema_url +POSTHOOK: Output: default@avro_extschema_url +PREHOOK: query: DESCRIBE avro_extschema_url +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url +POSTHOOK: query: DESCRIBE avro_extschema_url +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url +colx string +coly string +colz double +PREHOOK: query: ALTER TABLE avro_extschema_url UNSET TBLPROPERTIES ('avro.schema.url') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url +PREHOOK: Output: default@avro_extschema_url +POSTHOOK: query: ALTER TABLE avro_extschema_url UNSET TBLPROPERTIES ('avro.schema.url') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url +POSTHOOK: Output: default@avro_extschema_url +PREHOOK: query: DESCRIBE avro_extschema_url +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url +POSTHOOK: query: DESCRIBE avro_extschema_url +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url +colx string +coly string +colz double +PREHOOK: query: CREATE TABLE avro_extschema_url_parted + PARTITIONED BY (p1 string, p2 string) + STORED AS AVRO +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: CREATE TABLE avro_extschema_url_parted + PARTITIONED BY (p1 string, p2 string) + STORED AS AVRO +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_extschema_url_parted +PREHOOK: query: ALTER TABLE avro_extschema_url_parted + ADD PARTITION (p1=2017, p2=11) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted + ADD PARTITION (p1=2017, p2=11) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted@p1=2017/p2=11 +PREHOOK: query: ALTER TABLE avro_extschema_url_parted + ADD PARTITION (p1=2018, p2=2) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted + ADD PARTITION (p1=2018, p2=2) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted@p1=2018/p2=2 +PREHOOK: query: ALTER TABLE avro_extschema_url_parted + ADD PARTITION (p1=2018, p2=3) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted + ADD PARTITION (p1=2018, p2=3) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted@p1=2018/p2=3 +PREHOOK: query: DESCRIBE avro_extschema_url_parted +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +col1 string +col2 string +col3 double +col4 string +col5 string +col6 int +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: ALTER TABLE avro_extschema_url_parted SET +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url_parted +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted SET +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted +PREHOOK: query: ALTER TABLE avro_extschema_url_parted PARTITION (p1=2018) UPDATE COLUMNS +PREHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted PARTITION (p1=2018) UPDATE COLUMNS +POSTHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: Input: default@avro_extschema_url_parted@p1=2018/p2=2 +POSTHOOK: Input: default@avro_extschema_url_parted@p1=2018/p2=3 +POSTHOOK: Output: default@avro_extschema_url_parted@p1=2018/p2=2 +POSTHOOK: Output: default@avro_extschema_url_parted@p1=2018/p2=3 +PREHOOK: query: ALTER TABLE avro_extschema_url_parted UNSET TBLPROPERTIES ('avro.schema.url') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url_parted +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted UNSET TBLPROPERTIES ('avro.schema.url') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted +PREHOOK: query: DESCRIBE avro_extschema_url_parted +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +col1 string +col2 string +col3 double +col4 string +col5 string +col6 int +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2017, p2=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2017, p2=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +col1 string +col2 string +col3 double +col4 string +col5 string +col6 int +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=2) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=2) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +col1 string +col2 string +col3 double +col4 string +col5 string +col6 int +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=3) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=3) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +col1 string +col2 string +col3 double +col4 string +col5 string +col6 int +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: ALTER TABLE avro_extschema_url_parted SET +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url_parted +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted SET +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted +PREHOOK: query: ALTER TABLE avro_extschema_url_parted UPDATE COLUMNS +PREHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted UPDATE COLUMNS +POSTHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted +PREHOOK: query: ALTER TABLE avro_extschema_url_parted UNSET TBLPROPERTIES ('avro.schema.url') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url_parted +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted UNSET TBLPROPERTIES ('avro.schema.url') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted +PREHOOK: query: DESCRIBE avro_extschema_url_parted +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +colx string +coly string +colz double +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2017, p2=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2017, p2=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +colx string +coly string +colz double +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=2) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=2) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +colx string +coly string +colz double +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=3) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=3) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +colx string +coly string +colz double +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: ALTER TABLE avro_extschema_url_parted SET +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url_parted +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted SET +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted +PREHOOK: query: ALTER TABLE avro_extschema_url_parted PARTITION (p1=2017, p2=11) UPDATE COLUMNS +PREHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted PARTITION (p1=2017, p2=11) UPDATE COLUMNS +POSTHOOK: type: ALTERTABLE_UPDATECOLUMNS +POSTHOOK: Input: default@avro_extschema_url_parted@p1=2017/p2=11 +POSTHOOK: Output: default@avro_extschema_url_parted@p1=2017/p2=11 +PREHOOK: query: ALTER TABLE avro_extschema_url_parted UNSET TBLPROPERTIES ('avro.schema.url') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@avro_extschema_url_parted +PREHOOK: Output: default@avro_extschema_url_parted +POSTHOOK: query: ALTER TABLE avro_extschema_url_parted UNSET TBLPROPERTIES ('avro.schema.url') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: Output: default@avro_extschema_url_parted +PREHOOK: query: DESCRIBE avro_extschema_url_parted +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +colx string +coly string +colz double +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2017, p2=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2017, p2=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +colx string +coly string +colz double +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=2) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=2) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +colx string +coly string +colz double +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string +PREHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=3) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_extschema_url_parted +POSTHOOK: query: DESCRIBE avro_extschema_url_parted PARTITION (p1=2018, p2=3) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_extschema_url_parted +colx string +coly string +colz double +p1 string +p2 string + +# Partition Information +# col_name data_type comment +p1 string +p2 string