diff --git itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q index d4f0c7166f..e1c8636cde 100644 --- itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q +++ itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q @@ -36,7 +36,7 @@ PARTITIONED BY ( pt string, dt string, hr string) -SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +SKEWED BY (id) ON ('1', '2', '3') LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_list_bucketing'; INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id, name, dept FROM empty; diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out index ccd9ba5400..ce54a6d0d1 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out @@ -80,7 +80,7 @@ PARTITIONED BY ( pt string, dt string, hr string) -SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +SKEWED BY (id) ON ('1', '2', '3') #### A masked pattern was here #### PREHOOK: type: CREATETABLE PREHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing @@ -94,7 +94,7 @@ PARTITIONED BY ( pt string, dt string, hr string) -SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +SKEWED BY (id) ON ('1', '2', '3') #### A masked pattern was here #### POSTHOOK: type: CREATETABLE POSTHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java index 9b50fd4f30..5c21d84365 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java @@ -107,7 +107,7 @@ protected void setUp() { db.createTable(src, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); db.loadTable(hadoopDataFile[i], src, - LoadFileType.KEEP_EXISTING, false, false, false, false, null, 0, false); + LoadFileType.KEEP_EXISTING, false, false, false, null, 0, false); i++; } diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java index 3b22f152e8..cc86eb3271 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java @@ -644,10 +644,10 @@ public void testValidateLocations() throws Exception { "insert into CTLGS values(3, 'test_cat_2', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')", "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2')", "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'test_cat_2')", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n')", @@ -674,17 +674,17 @@ public void testValidateLocations() throws Exception { "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'test_cat_2')", "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'test_cat_2')", "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'test_cat_2')", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')", "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3000 ,1435255431,2,0 ,'hive',0,3000,'mytal3000','MANAGED_TABLE',NULL,NULL,'n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4011 ,1435255431,4,0 ,'hive',0,4001,'mytal4011','MANAGED_TABLE',NULL,NULL,'n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,4,0 ,'hive',0,4002,'','MANAGED_TABLE',NULL,NULL,'n')", @@ -773,8 +773,8 @@ private void createTestHiveTableSchemas() throws IOException { String[] scripts = new String[] { "insert into CTLGS values(2, 'my_catalog', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')", "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8021/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'my_catalog')", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,2,'aTable','MANAGED_TABLE',NULL,NULL,'n')", "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)" diff --git metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql index 8453ee382d..afdcc75b89 100644 --- metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql +++ metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql @@ -362,7 +362,6 @@ CREATE TABLE IF NOT EXISTS `SDS` ( `CD_ID` bigint, `INPUT_FORMAT` string, `IS_COMPRESSED` boolean, - `IS_STOREDASSUBDIRECTORIES` boolean, `LOCATION` string, `NUM_BUCKETS` int, `OUTPUT_FORMAT` string, @@ -378,7 +377,6 @@ TBLPROPERTIES ( \"CD_ID\", \"INPUT_FORMAT\", \"IS_COMPRESSED\", - \"IS_STOREDASSUBDIRECTORIES\", \"LOCATION\", \"NUM_BUCKETS\", \"OUTPUT_FORMAT\", diff --git metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql new file mode 100644 index 0000000000..d1be5edad4 --- /dev/null +++ metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql @@ -0,0 +1,1382 @@ +-- HIVE system db + +DROP DATABASE IF EXISTS SYS CASCADE; +CREATE DATABASE SYS; + +USE SYS; + +CREATE TABLE IF NOT EXISTS `BUCKETING_COLS` ( + `SD_ID` bigint, + `BUCKET_COL_NAME` string, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_BUCKETING_COLS` PRIMARY KEY (`SD_ID`,`INTEGER_IDX`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SD_ID\", + \"BUCKET_COL_NAME\", + \"INTEGER_IDX\" +FROM + \"BUCKETING_COLS\"" +); + +CREATE TABLE IF NOT EXISTS `CDS` ( + `CD_ID` bigint, + CONSTRAINT `SYS_PK_CDS` PRIMARY KEY (`CD_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CD_ID\" +FROM + \"CDS\"" +); + +CREATE TABLE IF NOT EXISTS `COLUMNS_V2` ( + `CD_ID` bigint, + `COMMENT` string, + `COLUMN_NAME` string, + `TYPE_NAME` string, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_COLUMN_V2` PRIMARY KEY (`CD_ID`,`COLUMN_NAME`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CD_ID\", + \"COMMENT\", + \"COLUMN_NAME\", + \"TYPE_NAME\", + \"INTEGER_IDX\" +FROM + \"COLUMNS_V2\"" +); + +CREATE TABLE IF NOT EXISTS `DATABASE_PARAMS` ( + `DB_ID` bigint, + `PARAM_KEY` string, + `PARAM_VALUE` string, + CONSTRAINT `SYS_PK_DATABASE_PARAMS` PRIMARY KEY (`DB_ID`,`PARAM_KEY`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"DB_ID\", + \"PARAM_KEY\", + \"PARAM_VALUE\" +FROM + \"DATABASE_PARAMS\"" +); + +CREATE TABLE IF NOT EXISTS `DBS` ( + `DB_ID` bigint, + `DB_LOCATION_URI` string, + `NAME` string, + `OWNER_NAME` string, + `OWNER_TYPE` string, + CONSTRAINT `SYS_PK_DBS` PRIMARY KEY (`DB_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"DB_ID\", + \"DB_LOCATION_URI\", + \"NAME\", + \"OWNER_NAME\", + \"OWNER_TYPE\" +FROM + \"DBS\"" +); + +CREATE TABLE IF NOT EXISTS `DB_PRIVS` ( + `DB_GRANT_ID` bigint, + `CREATE_TIME` int, + `DB_ID` bigint, + `GRANT_OPTION` int, + `GRANTOR` string, + `GRANTOR_TYPE` string, + `PRINCIPAL_NAME` string, + `PRINCIPAL_TYPE` string, + `DB_PRIV` string, + CONSTRAINT `SYS_PK_DB_PRIVS` PRIMARY KEY (`DB_GRANT_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"DB_GRANT_ID\", + \"CREATE_TIME\", + \"DB_ID\", + \"GRANT_OPTION\", + \"GRANTOR\", + \"GRANTOR_TYPE\", + \"PRINCIPAL_NAME\", + \"PRINCIPAL_TYPE\", + \"DB_PRIV\" +FROM + \"DB_PRIVS\"" +); + +CREATE TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( + `USER_GRANT_ID` bigint, + `CREATE_TIME` int, + `GRANT_OPTION` string, + `GRANTOR` string, + `GRANTOR_TYPE` string, + `PRINCIPAL_NAME` string, + `PRINCIPAL_TYPE` string, + `USER_PRIV` string, + CONSTRAINT `SYS_PK_GLOBAL_PRIVS` PRIMARY KEY (`USER_GRANT_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"USER_GRANT_ID\", + \"CREATE_TIME\", + \"GRANT_OPTION\", + \"GRANTOR\", + \"GRANTOR_TYPE\", + \"PRINCIPAL_NAME\", + \"PRINCIPAL_TYPE\", + \"USER_PRIV\" +FROM + \"GLOBAL_PRIVS\"" +); + +CREATE TABLE IF NOT EXISTS `PARTITIONS` ( + `PART_ID` bigint, + `CREATE_TIME` int, + `LAST_ACCESS_TIME` int, + `PART_NAME` string, + `SD_ID` bigint, + `TBL_ID` bigint, + CONSTRAINT `SYS_PK_PARTITIONS` PRIMARY KEY (`PART_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"PART_ID\", + \"CREATE_TIME\", + \"LAST_ACCESS_TIME\", + \"PART_NAME\", + \"SD_ID\", + \"TBL_ID\" +FROM + \"PARTITIONS\"" +); + +CREATE TABLE IF NOT EXISTS `PARTITION_KEYS` ( + `TBL_ID` bigint, + `PKEY_COMMENT` string, + `PKEY_NAME` string, + `PKEY_TYPE` string, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_PARTITION_KEYS` PRIMARY KEY (`TBL_ID`,`PKEY_NAME`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"TBL_ID\", + \"PKEY_COMMENT\", + \"PKEY_NAME\", + \"PKEY_TYPE\", + \"INTEGER_IDX\" +FROM + \"PARTITION_KEYS\"" +); + +CREATE TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( + `PART_ID` bigint, + `PART_KEY_VAL` string, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_PARTITION_KEY_VALS` PRIMARY KEY (`PART_ID`,`INTEGER_IDX`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"PART_ID\", + \"PART_KEY_VAL\", + \"INTEGER_IDX\" +FROM + \"PARTITION_KEY_VALS\"" +); + +CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` ( + `PART_ID` bigint, + `PARAM_KEY` string, + `PARAM_VALUE` string, + CONSTRAINT `SYS_PK_PARTITION_PARAMS` PRIMARY KEY (`PART_ID`,`PARAM_KEY`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"PART_ID\", + \"PARAM_KEY\", + \"PARAM_VALUE\" +FROM + \"PARTITION_PARAMS\"" +); + +CREATE TABLE IF NOT EXISTS `PART_COL_PRIVS` ( + `PART_COLUMN_GRANT_ID` bigint, + `COLUMN_NAME` string, + `CREATE_TIME` int, + `GRANT_OPTION` int, + `GRANTOR` string, + `GRANTOR_TYPE` string, + `PART_ID` bigint, + `PRINCIPAL_NAME` string, + `PRINCIPAL_TYPE` string, + `PART_COL_PRIV` string, + CONSTRAINT `SYS_PK_PART_COL_PRIVS` PRIMARY KEY (`PART_COLUMN_GRANT_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"PART_COLUMN_GRANT_ID\", + \"COLUMN_NAME\", + \"CREATE_TIME\", + \"GRANT_OPTION\", + \"GRANTOR\", + \"GRANTOR_TYPE\", + \"PART_ID\", + \"PRINCIPAL_NAME\", + \"PRINCIPAL_TYPE\", + \"PART_COL_PRIV\" +FROM + \"PART_COL_PRIVS\"" +); + +CREATE TABLE IF NOT EXISTS `PART_PRIVS` ( + `PART_GRANT_ID` bigint, + `CREATE_TIME` int, + `GRANT_OPTION` int, + `GRANTOR` string, + `GRANTOR_TYPE` string, + `PART_ID` bigint, + `PRINCIPAL_NAME` string, + `PRINCIPAL_TYPE` string, + `PART_PRIV` string, + CONSTRAINT `SYS_PK_PART_PRIVS` PRIMARY KEY (`PART_GRANT_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"PART_GRANT_ID\", + \"CREATE_TIME\", + \"GRANT_OPTION\", + \"GRANTOR\", + \"GRANTOR_TYPE\", + \"PART_ID\", + \"PRINCIPAL_NAME\", + \"PRINCIPAL_TYPE\", + \"PART_PRIV\" +FROM + \"PART_PRIVS\"" +); + +CREATE TABLE IF NOT EXISTS `ROLES` ( + `ROLE_ID` bigint, + `CREATE_TIME` int, + `OWNER_NAME` string, + `ROLE_NAME` string, + CONSTRAINT `SYS_PK_ROLES` PRIMARY KEY (`ROLE_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"ROLE_ID\", + \"CREATE_TIME\", + \"OWNER_NAME\", + \"ROLE_NAME\" +FROM + \"ROLES\"" +); + +CREATE TABLE IF NOT EXISTS `ROLE_MAP` ( + `ROLE_GRANT_ID` bigint, + `ADD_TIME` int, + `GRANT_OPTION` int, + `GRANTOR` string, + `GRANTOR_TYPE` string, + `PRINCIPAL_NAME` string, + `PRINCIPAL_TYPE` string, + `ROLE_ID` bigint, + CONSTRAINT `SYS_PK_ROLE_MAP` PRIMARY KEY (`ROLE_GRANT_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"ROLE_GRANT_ID\", + \"ADD_TIME\", + \"GRANT_OPTION\", + \"GRANTOR\", + \"GRANTOR_TYPE\", + \"PRINCIPAL_NAME\", + \"PRINCIPAL_TYPE\", + \"ROLE_ID\" +FROM + \"ROLE_MAP\"" +); + +CREATE TABLE IF NOT EXISTS `SDS` ( + `SD_ID` bigint, + `CD_ID` bigint, + `INPUT_FORMAT` string, + `IS_COMPRESSED` boolean, + `LOCATION` string, + `NUM_BUCKETS` int, + `OUTPUT_FORMAT` string, + `SERDE_ID` bigint, + CONSTRAINT `SYS_PK_SDS` PRIMARY KEY (`SD_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SD_ID\", + \"CD_ID\", + \"INPUT_FORMAT\", + \"IS_COMPRESSED\", + \"LOCATION\", + \"NUM_BUCKETS\", + \"OUTPUT_FORMAT\", + \"SERDE_ID\" +FROM + \"SDS\"" +); + +CREATE TABLE IF NOT EXISTS `SD_PARAMS` ( + `SD_ID` bigint, + `PARAM_KEY` string, + `PARAM_VALUE` string, + CONSTRAINT `SYS_PK_SD_PARAMS` PRIMARY KEY (`SD_ID`,`PARAM_KEY`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SD_ID\", + \"PARAM_KEY\", + \"PARAM_VALUE\" +FROM + \"SD_PARAMS\"" +); + +CREATE TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( + `SEQUENCE_NAME` string, + `NEXT_VAL` bigint, + CONSTRAINT `SYS_PK_SEQUENCE_TABLE` PRIMARY KEY (`SEQUENCE_NAME`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SEQUENCE_NAME\", + \"NEXT_VAL\" +FROM + \"SEQUENCE_TABLE\"" +); + +CREATE TABLE IF NOT EXISTS `SERDES` ( + `SERDE_ID` bigint, + `NAME` string, + `SLIB` string, + CONSTRAINT `SYS_PK_SERDES` PRIMARY KEY (`SERDE_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SERDE_ID\", + \"NAME\", + \"SLIB\" +FROM + \"SERDES\"" +); + +CREATE TABLE IF NOT EXISTS `SERDE_PARAMS` ( + `SERDE_ID` bigint, + `PARAM_KEY` string, + `PARAM_VALUE` string, + CONSTRAINT `SYS_PK_SERDE_PARAMS` PRIMARY KEY (`SERDE_ID`,`PARAM_KEY`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SERDE_ID\", + \"PARAM_KEY\", + \"PARAM_VALUE\" +FROM + \"SERDE_PARAMS\"" +); + +CREATE TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( + `SD_ID` bigint, + `SKEWED_COL_NAME` string, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_SKEWED_COL_NAMES` PRIMARY KEY (`SD_ID`,`INTEGER_IDX`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SD_ID\", + \"SKEWED_COL_NAME\", + \"INTEGER_IDX\" +FROM + \"SKEWED_COL_NAMES\"" +); + +CREATE TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( + `SD_ID` bigint, + `STRING_LIST_ID_KID` bigint, + `LOCATION` string, + CONSTRAINT `SYS_PK_COL_VALUE_LOC_MAP` PRIMARY KEY (`SD_ID`,`STRING_LIST_ID_KID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SD_ID\", + \"STRING_LIST_ID_KID\", + \"LOCATION\" +FROM + \"SKEWED_COL_VALUE_LOC_MAP\"" +); + +CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( + `STRING_LIST_ID` bigint, + CONSTRAINT `SYS_PK_SKEWED_STRING_LIST` PRIMARY KEY (`STRING_LIST_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"STRING_LIST_ID\" +FROM + \"SKEWED_STRING_LIST\"" +); + +CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( + `STRING_LIST_ID` bigint, + `STRING_LIST_VALUE` string, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_SKEWED_STRING_LIST_VALUES` PRIMARY KEY (`STRING_LIST_ID`,`INTEGER_IDX`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"STRING_LIST_ID\", + \"STRING_LIST_VALUE\", + \"INTEGER_IDX\" +FROM + \"SKEWED_STRING_LIST_VALUES\"" +); + +CREATE TABLE IF NOT EXISTS `SKEWED_VALUES` ( + `SD_ID_OID` bigint, + `STRING_LIST_ID_EID` bigint, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_SKEWED_VALUES` PRIMARY KEY (`SD_ID_OID`,`INTEGER_IDX`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SD_ID_OID\", + \"STRING_LIST_ID_EID\", + \"INTEGER_IDX\" +FROM + \"SKEWED_VALUES\"" +); + +CREATE TABLE IF NOT EXISTS `SORT_COLS` ( + `SD_ID` bigint, + `COLUMN_NAME` string, + `ORDER` int, + `INTEGER_IDX` int, + CONSTRAINT `SYS_PK_SORT_COLS` PRIMARY KEY (`SD_ID`,`INTEGER_IDX`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"SD_ID\", + \"COLUMN_NAME\", + \"ORDER\", + \"INTEGER_IDX\" +FROM + \"SORT_COLS\"" +); + +CREATE TABLE IF NOT EXISTS `TABLE_PARAMS` ( + `TBL_ID` bigint, + `PARAM_KEY` string, + `PARAM_VALUE` string, + CONSTRAINT `SYS_PK_TABLE_PARAMS` PRIMARY KEY (`TBL_ID`,`PARAM_KEY`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"TBL_ID\", + \"PARAM_KEY\", + \"PARAM_VALUE\" +FROM + \"TABLE_PARAMS\"" +); + +CREATE TABLE IF NOT EXISTS `TBLS` ( + `TBL_ID` bigint, + `CREATE_TIME` int, + `DB_ID` bigint, + `LAST_ACCESS_TIME` int, + `OWNER` string, + `RETENTION` int, + `SD_ID` bigint, + `TBL_NAME` string, + `TBL_TYPE` string, + `VIEW_EXPANDED_TEXT` string, + `VIEW_ORIGINAL_TEXT` string, + `IS_REWRITE_ENABLED` boolean, + CONSTRAINT `SYS_PK_TBLS` PRIMARY KEY (`TBL_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"TBL_ID\", + \"CREATE_TIME\", + \"DB_ID\", + \"LAST_ACCESS_TIME\", + \"OWNER\", + \"RETENTION\", + \"SD_ID\", + \"TBL_NAME\", + \"TBL_TYPE\", + \"VIEW_EXPANDED_TEXT\", + \"VIEW_ORIGINAL_TEXT\", + \"IS_REWRITE_ENABLED\" +FROM \"TBLS\"" +); + +CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( + `MV_CREATION_METADATA_ID` bigint, + `DB_NAME` string, + `TBL_NAME` string, + `TXN_LIST` string, + CONSTRAINT `SYS_PK_MV_CREATION_METADATA` PRIMARY KEY (`MV_CREATION_METADATA_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"MV_CREATION_METADATA_ID\", + \"DB_NAME\", + \"TBL_NAME\", + \"TXN_LIST\" +FROM \"MV_CREATION_METADATA\"" +); + +CREATE TABLE IF NOT EXISTS `MV_TABLES_USED` ( + `MV_CREATION_METADATA_ID` bigint, + `TBL_ID` bigint +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"MV_CREATION_METADATA_ID\", + \"TBL_ID\" +FROM \"MV_TABLES_USED\"" +); + +CREATE TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( + `TBL_COLUMN_GRANT_ID` bigint, + `COLUMN_NAME` string, + `CREATE_TIME` int, + `GRANT_OPTION` int, + `GRANTOR` string, + `GRANTOR_TYPE` string, + `PRINCIPAL_NAME` string, + `PRINCIPAL_TYPE` string, + `TBL_COL_PRIV` string, + `TBL_ID` bigint, + CONSTRAINT `SYS_PK_TBL_COL_PRIVS` PRIMARY KEY (`TBL_COLUMN_GRANT_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"TBL_COLUMN_GRANT_ID\", + \"COLUMN_NAME\", + \"CREATE_TIME\", + \"GRANT_OPTION\", + \"GRANTOR\", + \"GRANTOR_TYPE\", + \"PRINCIPAL_NAME\", + \"PRINCIPAL_TYPE\", + \"TBL_COL_PRIV\", + \"TBL_ID\" +FROM + \"TBL_COL_PRIVS\"" +); + +CREATE TABLE IF NOT EXISTS `TBL_PRIVS` ( + `TBL_GRANT_ID` bigint, + `CREATE_TIME` int, + `GRANT_OPTION` int, + `GRANTOR` string, + `GRANTOR_TYPE` string, + `PRINCIPAL_NAME` string, + `PRINCIPAL_TYPE` string, + `TBL_PRIV` string, + `TBL_ID` bigint, + CONSTRAINT `SYS_PK_TBL_PRIVS` PRIMARY KEY (`TBL_GRANT_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"TBL_GRANT_ID\", + \"CREATE_TIME\", + \"GRANT_OPTION\", + \"GRANTOR\", + \"GRANTOR_TYPE\", + \"PRINCIPAL_NAME\", + \"PRINCIPAL_TYPE\", + \"TBL_PRIV\", + \"TBL_ID\" +FROM + \"TBL_PRIVS\"" +); + +CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( + `CS_ID` bigint, + `DB_NAME` string, + `TABLE_NAME` string, + `COLUMN_NAME` string, + `COLUMN_TYPE` string, + `TBL_ID` bigint, + `LONG_LOW_VALUE` bigint, + `LONG_HIGH_VALUE` bigint, + `DOUBLE_HIGH_VALUE` double, + `DOUBLE_LOW_VALUE` double, + `BIG_DECIMAL_LOW_VALUE` string, + `BIG_DECIMAL_HIGH_VALUE` string, + `NUM_NULLS` bigint, + `NUM_DISTINCTS` bigint, + `AVG_COL_LEN` double, + `MAX_COL_LEN` bigint, + `NUM_TRUES` bigint, + `NUM_FALSES` bigint, + `LAST_ANALYZED` bigint, + CONSTRAINT `SYS_PK_TAB_COL_STATS` PRIMARY KEY (`CS_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CS_ID\", + \"DB_NAME\", + \"TABLE_NAME\", + \"COLUMN_NAME\", + \"COLUMN_TYPE\", + \"TBL_ID\", + \"LONG_LOW_VALUE\", + \"LONG_HIGH_VALUE\", + \"DOUBLE_HIGH_VALUE\", + \"DOUBLE_LOW_VALUE\", + \"BIG_DECIMAL_LOW_VALUE\", + \"BIG_DECIMAL_HIGH_VALUE\", + \"NUM_NULLS\", + \"NUM_DISTINCTS\", + \"AVG_COL_LEN\", + \"MAX_COL_LEN\", + \"NUM_TRUES\", + \"NUM_FALSES\", + \"LAST_ANALYZED\" +FROM + \"TAB_COL_STATS\"" +); + +CREATE TABLE IF NOT EXISTS `PART_COL_STATS` ( + `CS_ID` bigint, + `DB_NAME` string, + `TABLE_NAME` string, + `PARTITION_NAME` string, + `COLUMN_NAME` string, + `COLUMN_TYPE` string, + `PART_ID` bigint, + `LONG_LOW_VALUE` bigint, + `LONG_HIGH_VALUE` bigint, + `DOUBLE_HIGH_VALUE` double, + `DOUBLE_LOW_VALUE` double, + `BIG_DECIMAL_LOW_VALUE` string, + `BIG_DECIMAL_HIGH_VALUE` string, + `NUM_NULLS` bigint, + `NUM_DISTINCTS` bigint, + `AVG_COL_LEN` double, + `MAX_COL_LEN` bigint, + `NUM_TRUES` bigint, + `NUM_FALSES` bigint, + `LAST_ANALYZED` bigint, + CONSTRAINT `SYS_PK_PART_COL_STATS` PRIMARY KEY (`CS_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CS_ID\", + \"DB_NAME\", + \"TABLE_NAME\", + \"PARTITION_NAME\", + \"COLUMN_NAME\", + \"COLUMN_TYPE\", + \"PART_ID\", + \"LONG_LOW_VALUE\", + \"LONG_HIGH_VALUE\", + \"DOUBLE_HIGH_VALUE\", + \"DOUBLE_LOW_VALUE\", + \"BIG_DECIMAL_LOW_VALUE\", + \"BIG_DECIMAL_HIGH_VALUE\", + \"NUM_NULLS\", + \"NUM_DISTINCTS\", + \"AVG_COL_LEN\", + \"MAX_COL_LEN\", + \"NUM_TRUES\", + \"NUM_FALSES\", + \"LAST_ANALYZED\" +FROM + \"PART_COL_STATS\"" +); + +CREATE TABLE IF NOT EXISTS `VERSION` ( + `VER_ID` BIGINT, + `SCHEMA_VERSION` string, + `VERSION_COMMENT` string, + CONSTRAINT `SYS_PK_VERSION` PRIMARY KEY (`VER_ID`) DISABLE +); + +INSERT INTO `VERSION` VALUES (1, '3.0.0', 'Hive release version 3.0.0'); + +CREATE TABLE IF NOT EXISTS `DB_VERSION` ( + `VER_ID` BIGINT, + `SCHEMA_VERSION` string, + `VERSION_COMMENT` string, + CONSTRAINT `SYS_PK_DB_VERSION` PRIMARY KEY (`VER_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"VER_ID\", + \"SCHEMA_VERSION\", + \"VERSION_COMMENT\" +FROM + \"VERSION\"" +); + +CREATE TABLE IF NOT EXISTS `FUNCS` ( + `FUNC_ID` bigint, + `CLASS_NAME` string, + `CREATE_TIME` int, + `DB_ID` bigint, + `FUNC_NAME` string, + `FUNC_TYPE` int, + `OWNER_NAME` string, + `OWNER_TYPE` string, + CONSTRAINT `SYS_PK_FUNCS` PRIMARY KEY (`FUNC_ID`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"FUNC_ID\", + \"CLASS_NAME\", + \"CREATE_TIME\", + \"DB_ID\", + \"FUNC_NAME\", + \"FUNC_TYPE\", + \"OWNER_NAME\", + \"OWNER_TYPE\" +FROM + \"FUNCS\"" +); + +-- CREATE TABLE IF NOT EXISTS `FUNC_RU` ( +-- `FUNC_ID` bigint, +-- `RESOURCE_TYPE` int, +-- `RESOURCE_URI` string, +-- `INTEGER_IDX` int, +-- CONSTRAINT `SYS_PK_FUNCS_RU` PRIMARY KEY (`FUNC_ID`, `INTEGER_IDX`) DISABLE +-- ) +-- STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +-- TBLPROPERTIES ( +-- "hive.sql.database.type" = "METASTORE", +-- "hive.sql.query" = "SELECT * FROM FUNCS_RU" +-- ); + +CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS` +( + `CHILD_CD_ID` bigint, + `CHILD_INTEGER_IDX` int, + `CHILD_TBL_ID` bigint, + `PARENT_CD_ID` bigint, + `PARENT_INTEGER_IDX` int, + `PARENT_TBL_ID` bigint, + `POSITION` bigint, + `CONSTRAINT_NAME` string, + `CONSTRAINT_TYPE` string, + `UPDATE_RULE` string, + `DELETE_RULE` string, + `ENABLE_VALIDATE_RELY` int, + `DEFAULT_VALUE` string, + CONSTRAINT `SYS_PK_KEY_CONSTRAINTS` PRIMARY KEY (`CONSTRAINT_NAME`, `POSITION`) DISABLE +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"CHILD_CD_ID\", + \"CHILD_INTEGER_IDX\", + \"CHILD_TBL_ID\", + \"PARENT_CD_ID\", + \"PARENT_INTEGER_IDX\", + \"PARENT_TBL_ID\", + \"POSITION\", + \"CONSTRAINT_NAME\", + \"CONSTRAINT_TYPE\", + \"UPDATE_RULE\", + \"DELETE_RULE\", + \"ENABLE_VALIDATE_RELY\", + \"DEFAULT_VALUE\" +FROM + \"KEY_CONSTRAINTS\"" +); + +CREATE VIEW `TABLE_STATS_VIEW` AS +SELECT + `TBL_ID`, + max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE, + max(CASE `PARAM_KEY` WHEN 'numFiles' THEN `PARAM_VALUE` END) AS NUM_FILES, + max(CASE `PARAM_KEY` WHEN 'numRows' THEN `PARAM_VALUE` END) AS NUM_ROWS, + max(CASE `PARAM_KEY` WHEN 'rawDataSize' THEN `PARAM_VALUE` END) AS RAW_DATA_SIZE, + max(CASE `PARAM_KEY` WHEN 'totalSize' THEN `PARAM_VALUE` END) AS TOTAL_SIZE, + max(CASE `PARAM_KEY` WHEN 'transient_lastDdlTime' THEN `PARAM_VALUE` END) AS TRANSIENT_LAST_DDL_TIME +FROM `TABLE_PARAMS` GROUP BY `TBL_ID`; + +CREATE VIEW `PARTITION_STATS_VIEW` AS +SELECT + `PART_ID`, + max(CASE `PARAM_KEY` WHEN 'COLUMN_STATS_ACCURATE' THEN `PARAM_VALUE` END) AS COLUMN_STATS_ACCURATE, + max(CASE `PARAM_KEY` WHEN 'numFiles' THEN `PARAM_VALUE` END) AS NUM_FILES, + max(CASE `PARAM_KEY` WHEN 'numRows' THEN `PARAM_VALUE` END) AS NUM_ROWS, + max(CASE `PARAM_KEY` WHEN 'rawDataSize' THEN `PARAM_VALUE` END) AS RAW_DATA_SIZE, + max(CASE `PARAM_KEY` WHEN 'totalSize' THEN `PARAM_VALUE` END) AS TOTAL_SIZE, + max(CASE `PARAM_KEY` WHEN 'transient_lastDdlTime' THEN `PARAM_VALUE` END) AS TRANSIENT_LAST_DDL_TIME +FROM `PARTITION_PARAMS` GROUP BY `PART_ID`; + +CREATE TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( + `NAME` string, + `STATUS` string, + `QUERY_PARALLELISM` int, + `DEFAULT_POOL_PATH` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"WM_RESOURCEPLAN\".\"NAME\", + \"STATUS\", + \"WM_RESOURCEPLAN\".\"QUERY_PARALLELISM\", + \"WM_POOL\".\"PATH\" +FROM + \"WM_RESOURCEPLAN\" LEFT OUTER JOIN \"WM_POOL\" ON \"WM_RESOURCEPLAN\".\"DEFAULT_POOL_ID\" = \"WM_POOL\".\"POOL_ID\"" +); + +CREATE TABLE IF NOT EXISTS `WM_TRIGGERS` ( + `RP_NAME` string, + `NAME` string, + `TRIGGER_EXPRESSION` string, + `ACTION_EXPRESSION` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + r.\"NAME\" AS RP_NAME, + t.\"NAME\" AS NAME, + \"TRIGGER_EXPRESSION\", + \"ACTION_EXPRESSION\" +FROM + \"WM_TRIGGER\" t +JOIN + \"WM_RESOURCEPLAN\" r +ON + t.\"RP_ID\" = r.\"RP_ID\"" +); + +CREATE TABLE IF NOT EXISTS `WM_POOLS` ( + `RP_NAME` string, + `PATH` string, + `ALLOC_FRACTION` double, + `QUERY_PARALLELISM` int, + `SCHEDULING_POLICY` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"WM_RESOURCEPLAN\".\"NAME\", + \"WM_POOL\".\"PATH\", + \"WM_POOL\".\"ALLOC_FRACTION\", + \"WM_POOL\".\"QUERY_PARALLELISM\", + \"WM_POOL\".\"SCHEDULING_POLICY\" +FROM + \"WM_POOL\" +JOIN + \"WM_RESOURCEPLAN\" +ON + \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"" +); + +CREATE TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( + `RP_NAME` string, + `POOL_PATH` string, + `TRIGGER_NAME` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + \"WM_POOL\".\"PATH\" AS POOL_PATH, + \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME +FROM \"WM_POOL_TO_TRIGGER\" + JOIN \"WM_POOL\" ON \"WM_POOL_TO_TRIGGER\".\"POOL_ID\" = \"WM_POOL\".\"POOL_ID\" + JOIN \"WM_TRIGGER\" ON \"WM_POOL_TO_TRIGGER\".\"TRIGGER_ID\" = \"WM_TRIGGER\".\"TRIGGER_ID\" + JOIN \"WM_RESOURCEPLAN\" ON \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +UNION +SELECT + \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + '' AS POOL_PATH, + \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME +FROM \"WM_TRIGGER\" + JOIN \"WM_RESOURCEPLAN\" ON \"WM_TRIGGER\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't') +" +); + +CREATE TABLE IF NOT EXISTS `WM_MAPPINGS` ( + `RP_NAME` string, + `ENTITY_TYPE` string, + `ENTITY_NAME` string, + `POOL_PATH` string, + `ORDERING` int +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"WM_RESOURCEPLAN\".\"NAME\", + \"ENTITY_TYPE\", + \"ENTITY_NAME\", + case when \"WM_POOL\".\"PATH\" is null then '' else \"WM_POOL\".\"PATH\" end, + \"ORDERING\" +FROM \"WM_MAPPING\" +JOIN \"WM_RESOURCEPLAN\" ON \"WM_MAPPING\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_ID\" +" +); + +DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE; +CREATE DATABASE INFORMATION_SCHEMA; + +USE INFORMATION_SCHEMA; + +CREATE VIEW IF NOT EXISTS `SCHEMATA` +( + `CATALOG_NAME`, + `SCHEMA_NAME`, + `SCHEMA_OWNER`, + `DEFAULT_CHARACTER_SET_CATALOG`, + `DEFAULT_CHARACTER_SET_SCHEMA`, + `DEFAULT_CHARACTER_SET_NAME`, + `SQL_PATH` +) AS +SELECT DISTINCT + 'default', + D.`NAME`, + D.`OWNER_NAME`, + cast(null as string), + cast(null as string), + cast(null as string), + `DB_LOCATION_URI` +FROM + `sys`.`DBS` D, `sys`.`TBLS` T, `sys`.`TBL_PRIVS` P +WHERE + NOT restrict_information_schema() OR + D.`DB_ID` = T.`DB_ID` + AND T.`TBL_ID` = P.`TBL_ID` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')); + +CREATE VIEW IF NOT EXISTS `TABLES` +( + `TABLE_CATALOG`, + `TABLE_SCHEMA`, + `TABLE_NAME`, + `TABLE_TYPE`, + `SELF_REFERENCING_COLUMN_NAME`, + `REFERENCE_GENERATION`, + `USER_DEFINED_TYPE_CATALOG`, + `USER_DEFINED_TYPE_SCHEMA`, + `USER_DEFINED_TYPE_NAME`, + `IS_INSERTABLE_INTO`, + `IS_TYPED`, + `COMMIT_ACTION` +) AS +SELECT DISTINCT + 'default', + D.NAME, + T.TBL_NAME, + IF(length(T.VIEW_ORIGINAL_TEXT) > 0, 'VIEW', 'BASE_TABLE'), + cast(null as string), + cast(null as string), + cast(null as string), + cast(null as string), + cast(null as string), + IF(length(T.VIEW_ORIGINAL_TEXT) > 0, 'NO', 'YES'), + 'NO', + cast(null as string) +FROM + `sys`.`TBLS` T, `sys`.`DBS` D, `sys`.`TBL_PRIVS` P +WHERE + D.`DB_ID` = T.`DB_ID` + AND (NOT restrict_information_schema() OR T.`TBL_ID` = P.`TBL_ID` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + AND P.`TBL_PRIV`='SELECT'); + +CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES` +( + `GRANTOR`, + `GRANTEE`, + `TABLE_CATALOG`, + `TABLE_SCHEMA`, + `TABLE_NAME`, + `PRIVILEGE_TYPE`, + `IS_GRANTABLE`, + `WITH_HIERARCHY` +) AS +SELECT DISTINCT + P.`GRANTOR`, + P.`PRINCIPAL_NAME`, + 'default', + D.`NAME`, + T.`TBL_NAME`, + P.`TBL_PRIV`, + IF (P.`GRANT_OPTION` == 0, 'NO', 'YES'), + 'NO' +FROM + `sys`.`TBL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_PRIVS` P2 +WHERE + P.`TBL_ID` = T.`TBL_ID` + AND T.`DB_ID` = D.`DB_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) + AND P2.`TBL_PRIV`='SELECT'); + +CREATE VIEW IF NOT EXISTS `COLUMNS` +( + `TABLE_CATALOG`, + `TABLE_SCHEMA`, + `TABLE_NAME`, + `COLUMN_NAME`, + `ORDINAL_POSITION`, + `COLUMN_DEFAULT`, + `IS_NULLABLE`, + `DATA_TYPE`, + `CHARACTER_MAXIMUM_LENGTH`, + `CHARACTER_OCTET_LENGTH`, + `NUMERIC_PRECISION`, + `NUMERIC_PRECISION_RADIX`, + `NUMERIC_SCALE`, + `DATETIME_PRECISION`, + `INTERVAL_TYPE`, + `INTERVAL_PRECISION`, + `CHARACTER_SET_CATALOG`, + `CHARACTER_SET_SCHEMA`, + `CHARACTER_SET_NAME`, + `COLLATION_CATALOG`, + `COLLATION_SCHEMA`, + `COLLATION_NAME`, + `UDT_CATALOG`, + `UDT_SCHEMA`, + `UDT_NAME`, + `SCOPE_CATALOG`, + `SCOPE_SCHEMA`, + `SCOPE_NAME`, + `MAXIMUM_CARDINALITY`, + `DTD_IDENTIFIER`, + `IS_SELF_REFERENCING`, + `IS_IDENTITY`, + `IDENTITY_GENERATION`, + `IDENTITY_START`, + `IDENTITY_INCREMENT`, + `IDENTITY_MAXIMUM`, + `IDENTITY_MINIMUM`, + `IDENTITY_CYCLE`, + `IS_GENERATED`, + `GENERATION_EXPRESSION`, + `IS_SYSTEM_TIME_PERIOD_START`, + `IS_SYSTEM_TIME_PERIOD_END`, + `SYSTEM_TIME_PERIOD_TIMESTAMP_GENERATION`, + `IS_UPDATABLE`, + `DECLARED_DATA_TYPE`, + `DECLARED_NUMERIC_PRECISION`, + `DECLARED_NUMERIC_SCALE` +) AS +SELECT DISTINCT + 'default', + D.NAME, + T.TBL_NAME, + C.COLUMN_NAME, + C.INTEGER_IDX, + cast (null as string), + 'YES', + C.TYPE_NAME as TYPE_NAME, + CASE WHEN lower(C.TYPE_NAME) like 'varchar%' THEN cast(regexp_extract(upper(C.TYPE_NAME), '^VARCHAR\\s*\\((\\d+)\\s*\\)$', 1) as int) + WHEN lower(C.TYPE_NAME) like 'char%' THEN cast(regexp_extract(upper(C.TYPE_NAME), '^CHAR\\s*\\((\\d+)\\s*\\)$', 1) as int) + ELSE null END, + CASE WHEN lower(C.TYPE_NAME) like 'varchar%' THEN cast(regexp_extract(upper(C.TYPE_NAME), '^VARCHAR\\s*\\((\\d+)\\s*\\)$', 1) as int) + WHEN lower(C.TYPE_NAME) like 'char%' THEN cast(regexp_extract(upper(C.TYPE_NAME), '^CHAR\\s*\\((\\d+)\\s*\\)$', 1) as int) + ELSE null END, + CASE WHEN lower(C.TYPE_NAME) = 'bigint' THEN 19 + WHEN lower(C.TYPE_NAME) = 'int' THEN 10 + WHEN lower(C.TYPE_NAME) = 'smallint' THEN 5 + WHEN lower(C.TYPE_NAME) = 'tinyint' THEN 3 + WHEN lower(C.TYPE_NAME) = 'float' THEN 23 + WHEN lower(C.TYPE_NAME) = 'double' THEN 53 + WHEN lower(C.TYPE_NAME) like 'decimal%' THEN regexp_extract(upper(C.TYPE_NAME), '^DECIMAL\\s*\\((\\d+)',1) + WHEN lower(C.TYPE_NAME) like 'numeric%' THEN regexp_extract(upper(C.TYPE_NAME), '^NUMERIC\\s*\\((\\d+)',1) + ELSE null END, + CASE WHEN lower(C.TYPE_NAME) = 'bigint' THEN 10 + WHEN lower(C.TYPE_NAME) = 'int' THEN 10 + WHEN lower(C.TYPE_NAME) = 'smallint' THEN 10 + WHEN lower(C.TYPE_NAME) = 'tinyint' THEN 10 + WHEN lower(C.TYPE_NAME) = 'float' THEN 2 + WHEN lower(C.TYPE_NAME) = 'double' THEN 2 + WHEN lower(C.TYPE_NAME) like 'decimal%' THEN 10 + WHEN lower(C.TYPE_NAME) like 'numeric%' THEN 10 + ELSE null END, + CASE WHEN lower(C.TYPE_NAME) like 'decimal%' THEN regexp_extract(upper(C.TYPE_NAME), '^DECIMAL\\s*\\((\\d+),(\\d+)',2) + WHEN lower(C.TYPE_NAME) like 'numeric%' THEN regexp_extract(upper(C.TYPE_NAME), '^NUMERIC\\s*\\((\\d+),(\\d+)',2) + ELSE null END, + CASE WHEN lower(C.TYPE_NAME) = 'date' THEN 0 + WHEN lower(C.TYPE_NAME) = 'timestamp' THEN 9 + ELSE null END, + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + C.CD_ID, + 'NO', + 'NO', + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + cast (null as string), + 'NEVER', + cast (null as string), + 'NO', + 'NO', + cast (null as string), + 'YES', + C.TYPE_NAME as DECLARED_DATA_TYPE, + CASE WHEN lower(C.TYPE_NAME) = 'bigint' THEN 19 + WHEN lower(C.TYPE_NAME) = 'int' THEN 10 + WHEN lower(C.TYPE_NAME) = 'smallint' THEN 5 + WHEN lower(C.TYPE_NAME) = 'tinyint' THEN 3 + WHEN lower(C.TYPE_NAME) = 'float' THEN 23 + WHEN lower(C.TYPE_NAME) = 'double' THEN 53 + WHEN lower(C.TYPE_NAME) like 'decimal%' THEN regexp_extract(upper(C.TYPE_NAME), '^DECIMAL\\s*\\((\\d+)',1) + WHEN lower(C.TYPE_NAME) like 'numeric%' THEN regexp_extract(upper(C.TYPE_NAME), '^NUMERIC\\s*\\((\\d+)',1) + ELSE null END, + CASE WHEN lower(C.TYPE_NAME) = 'bigint' THEN 10 + WHEN lower(C.TYPE_NAME) = 'int' THEN 10 + WHEN lower(C.TYPE_NAME) = 'smallint' THEN 10 + WHEN lower(C.TYPE_NAME) = 'tinyint' THEN 10 + WHEN lower(C.TYPE_NAME) = 'float' THEN 2 + WHEN lower(C.TYPE_NAME) = 'double' THEN 2 + WHEN lower(C.TYPE_NAME) like 'decimal%' THEN 10 + WHEN lower(C.TYPE_NAME) like 'numeric%' THEN 10 + ELSE null END +FROM + `sys`.`COLUMNS_V2` C, + `sys`.`SDS` S, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_COL_PRIVS` P +WHERE + S.`SD_ID` = T.`SD_ID` + AND T.`DB_ID` = D.`DB_ID` + AND C.`CD_ID` = S.`CD_ID` + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` + AND C.`COLUMN_NAME` = P.`COLUMN_NAME` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + AND P.`TBL_COL_PRIV`='SELECT'); + +CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES` +( + `GRANTOR`, + `GRANTEE`, + `TABLE_CATALOG`, + `TABLE_SCHEMA`, + `TABLE_NAME`, + `COLUMN_NAME`, + `PRIVILEGE_TYPE`, + `IS_GRANTABLE` +) AS +SELECT DISTINCT + P.`GRANTOR`, + P.`PRINCIPAL_NAME`, + 'default', + D.`NAME`, + T.`TBL_NAME`, + P.`COLUMN_NAME`, + P.`TBL_COL_PRIV`, + IF (P.`GRANT_OPTION` == 0, 'NO', 'YES') +FROM + `sys`.`TBL_COL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`SDS` S, + `sys`.`TBL_PRIVS` P2 +WHERE + S.`SD_ID` = T.`SD_ID` + AND T.`DB_ID` = D.`DB_ID` + AND P.`TBL_ID` = T.`TBL_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) + AND P2.`TBL_PRIV`='SELECT'); + +CREATE VIEW IF NOT EXISTS `VIEWS` +( + `TABLE_CATALOG`, + `TABLE_SCHEMA`, + `TABLE_NAME`, + `VIEW_DEFINITION`, + `CHECK_OPTION`, + `IS_UPDATABLE`, + `IS_INSERTABLE_INTO`, + `IS_TRIGGER_UPDATABLE`, + `IS_TRIGGER_DELETABLE`, + `IS_TRIGGER_INSERTABLE_INTO` +) AS +SELECT DISTINCT + 'default', + D.NAME, + T.TBL_NAME, + T.VIEW_ORIGINAL_TEXT, + CAST(NULL as string), + false, + false, + false, + false, + false +FROM + `sys`.`DBS` D, + `sys`.`TBLS` T, + `sys`.`TBL_PRIVS` P +WHERE + D.`DB_ID` = T.`DB_ID` + AND length(T.VIEW_ORIGINAL_TEXT) > 0 + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + AND P.`TBL_PRIV`='SELECT'); diff --git ql/pom.xml ql/pom.xml index 0c181e515c..ba15cd47db 100644 --- ql/pom.xml +++ ql/pom.xml @@ -819,7 +819,7 @@ ${basedir}/src/java ${basedir}/src/test - + org.antlr antlr3-maven-plugin diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 90d6b8f4a3..eced035a3d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -246,8 +246,6 @@ INVALID_TABLE_IN_ON_CLAUSE_OF_MERGE(10149, "No columns from target table ''{0}'' found in ON " + "clause ''{1}'' of MERGE statement.", true), - LOAD_INTO_STORED_AS_DIR(10195, "A stored-as-directories table cannot be used as target for LOAD"), - ALTER_TBL_STOREDASDIR_NOT_SKEWED(10196, "This operation is only valid on skewed table."), ALTER_TBL_SKEWED_LOC_NO_LOC(10197, "Alter table skewed location doesn't have locations."), ALTER_TBL_SKEWED_LOC_NO_MAP(10198, "Alter table skewed location doesn't have location map."), SKEWED_TABLE_NO_COLUMN_NAME(10200, "No skewed column name."), diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 8e32b02b59..d861f774ce 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -987,7 +987,6 @@ private DataOutputStream getOutputStream(Path outputFile) throws HiveException { private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc, DriverContext driverContext) throws HiveException { ListBucketingCtx lbCtx = mergeFilesDesc.getLbCtx(); - boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel(); // merge work only needs input and output. @@ -1000,7 +999,6 @@ private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc, pathToAliases.put(mergeFilesDesc.getInputDir().get(0), inputDirstr); mergeWork.setPathToAliases(pathToAliases); mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx()); - mergeWork.resolveConcatenateMerge(db.getConf()); mergeWork.setMapperCannotSpanPartns(true); mergeWork.setSourceTableInputFormat(mergeFilesDesc.getInputFormatClass().getName()); final FileMergeDesc fmd; @@ -1013,7 +1011,7 @@ private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc, fmd.setDpCtx(null); fmd.setHasDynamicPartitions(false); - fmd.setListBucketingAlterTableConcatenate(lbatc); + fmd.setListBucketingAlterTableConcatenate(false); fmd.setListBucketingDepth(lbd); fmd.setOutputPath(mergeFilesDesc.getOutputDir()); @@ -2673,9 +2671,6 @@ else if (sortCol.getOrder() == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_DESC) { colValueList.add("('" + StringUtils.join(colValues, "','") + "')"); } tbl_skewedinfo.append(StringUtils.join(colValueList, ",") + ")"); - if (tbl.isStoredAsSubDirectories()) { - tbl_skewedinfo.append("\n STORED AS DIRECTORIES"); - } } // Row format (SerDe) @@ -4328,8 +4323,6 @@ private static StorageDescriptor retrieveStorageDescriptor(Table tbl, Partition tbl.setSkewedColNames(skewedColNames); tbl.setSkewedColValues(skewedValues); } - - tbl.setStoredAsSubDirectories(alterTbl.isStoredAsSubDirectories()); } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.OWNER) { if (alterTbl.getOwnerPrincipal() != null) { tbl.setOwner(alterTbl.getOwnerPrincipal().getName()); @@ -4405,24 +4398,6 @@ private static StorageDescriptor retrieveStorageDescriptor(Table tbl, Partition return result; } - private void checkMmLb(Table tbl) throws HiveException { - if (!tbl.isStoredAsSubDirectories()) { - return; - } - // TODO [MM gap?]: by design; no-one seems to use LB tables. They will work, but not convert. - // It's possible to work around this by re-creating and re-inserting the table. - throw new HiveException("Converting list bucketed tables stored as subdirectories " - + " to MM is not supported. Please re-create a table in the desired format."); - } - - private void checkMmLb(Partition part) throws HiveException { - if (!part.isStoredAsSubDirectories()) { - return; - } - throw new HiveException("Converting list bucketed tables stored as subdirectories " - + " to MM is not supported. Please re-create a table in the desired format."); - } - private List> generateAddMmTasks(Table tbl, Long writeId) throws HiveException { // We will move all the files in the table/partition directories into the first MM // directory, then commit the first write ID. @@ -4441,7 +4416,6 @@ private void checkMmLb(Partition part) throws HiveException { Iterator partIter = parts.iterator(); while (partIter.hasNext()) { Partition part = partIter.next(); - checkMmLb(part); Path src = part.getDataLocation(), tgt = new Path(src, mmDir); srcs.add(src); tgts.add(tgt); @@ -4450,7 +4424,6 @@ private void checkMmLb(Partition part) throws HiveException { } } } else { - checkMmLb(tbl); Path src = tbl.getDataLocation(), tgt = new Path(src, mmDir); srcs.add(src); tgts.add(tgt); @@ -4480,19 +4453,6 @@ private void checkMmLb(Partition part) throws HiveException { if (!isFromMmTable && isToMmTable) { if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_MM_ALLOW_ORIGINALS)) { result = generateAddMmTasks(tbl, alterTbl.getWriteId()); - } else { - if (tbl.getPartitionKeys().size() > 0) { - Hive db = getHive(); - PartitionIterable parts = new PartitionIterable(db, tbl, null, - HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX)); - Iterator partIter = parts.iterator(); - while (partIter.hasNext()) { - Partition part0 = partIter.next(); - checkMmLb(part0); - } - } else { - checkMmLb(tbl); - } } } else if (isFromMmTable && !isToMmTable) { throw new HiveException("Cannot convert an ACID table to non-ACID"); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index c2319bb960..5370707e87 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -32,8 +32,6 @@ import java.util.Set; import java.util.function.BiFunction; -import com.google.common.collect.Lists; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -42,8 +40,8 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -71,7 +69,10 @@ import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.stats.StatsCollectionContext; import org.apache.hadoop.hive.ql.stats.StatsPublisher; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.Serializer; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; @@ -92,8 +93,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - - /** * File Sink operator implementation. **/ @@ -120,7 +119,6 @@ protected transient RecordWriter[] rowOutWriters; // row specific RecordWriters protected transient int maxPartitions; protected transient ListBucketingCtx lbCtx; - protected transient boolean isSkewedStoredAsSubDirectories; protected transient boolean[] statsFromRecordWriter; protected transient boolean isCollectRWStats; private transient FSPaths prevFsp; @@ -233,7 +231,7 @@ private void commit(FileSystem fs, List commitPaths) throws HiveException private void commitOneOutPath(int idx, FileSystem fs, List commitPaths) throws IOException, HiveException { - if ((bDynParts || isSkewedStoredAsSubDirectories) + if ((bDynParts) && !fs.exists(finalPaths[idx].getParent())) { if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("commit making path for dyn/skew: " + finalPaths[idx].getParent()); @@ -284,13 +282,12 @@ public void abortWriters(FileSystem fs, boolean abort, boolean delete) throws Hi } } - public void initializeBucketPaths(int filesIdx, String taskId, boolean isNativeTable, - boolean isSkewedStoredAsSubDirectories) { + public void initializeBucketPaths(int filesIdx, String taskId, boolean isNativeTable) { if (isNativeTable) { String extension = Utilities.getFileExtension(jc, isCompressed, hiveOutputFormat); String taskWithExt = extension == null ? taskId : taskId + extension; if (!isMmTable) { - if (!bDynParts && !isSkewedStoredAsSubDirectories) { + if (!bDynParts) { finalPaths[filesIdx] = new Path(parent, taskWithExt); } else { finalPaths[filesIdx] = new Path(buildTmpPath(), taskWithExt); @@ -519,13 +516,9 @@ protected void initializeOp(Configuration hconf) throws HiveException { dpSetup(); } - if (lbCtx != null) { - lbSetup(); - } - if (!bDynParts) { fsp = new FSPaths(specPath, conf.isMmTable()); - fsp.subdirAfterTxn = combinePathFragments(generateListBucketingDirName(null), unionPath); + fsp.subdirAfterTxn = unionPath; if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("creating new paths " + System.identityHashCode(fsp) + " from ctor; childSpec " + unionPath + ": tmpPath " + fsp.buildTmpPath() @@ -535,9 +528,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { // Create all the files - this is required because empty files need to be created for // empty buckets // createBucketFiles(fsp); - if (!this.isSkewedStoredAsSubDirectories) { - valToPaths.put("", fsp); // special entry for non-DP case - } + valToPaths.put("", fsp); } final StoragePolicyValue tmpStorage = StoragePolicyValue.lookup(HiveConf @@ -615,13 +606,6 @@ private void logOutputFormatError(Configuration hconf, HiveException ex) { LOG.error(errorWriter.toString(), ex); } - /** - * Initialize list bucketing information - */ - private void lbSetup() { - this.isSkewedStoredAsSubDirectories = ((lbCtx == null) ? false : lbCtx.isSkewedStoredAsDir()); - } - /** * Set up for dynamic partitioning including a new ObjectInspector for the output row. */ @@ -713,7 +697,7 @@ protected void createBucketFiles(FSPaths fsp) throws HiveException { protected void createBucketForFileIdx(FSPaths fsp, int filesIdx) throws HiveException { try { - fsp.initializeBucketPaths(filesIdx, taskId, isNativeTable(), isSkewedStoredAsSubDirectories); + fsp.initializeBucketPaths(filesIdx, taskId, isNativeTable()); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("createBucketForFileIdx " + filesIdx + ": final path " + fsp.finalPaths[filesIdx] + "; out path " + fsp.outPaths[filesIdx] +" (spec path " + specPath + ", tmp path " @@ -849,18 +833,9 @@ protected boolean updateProgress() { @Override public void process(Object row, int tag) throws HiveException { runTimeNumRows++; - /* Create list bucketing sub-directory only if stored-as-directories is on. */ - String lbDirName = null; - lbDirName = (lbCtx == null) ? null : generateListBucketingDirName(row); if (!bDynParts && !filesCreated) { - if (lbDirName != null) { - if (valToPaths.get(lbDirName) == null) { - createNewPaths(null, lbDirName); - } - } else { - createBucketFiles(fsp); - } + createBucketFiles(fsp); } try { @@ -900,19 +875,13 @@ public void process(Object row, int tag) throws HiveException { dpCtx.getWhiteListPattern().toString() + "'. " + "(configure with " + HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.varname + ")"); } - fpaths = getDynOutPaths(dpVals, lbDirName); + fpaths = getDynOutPaths(dpVals); // use SubStructObjectInspector to serialize the non-partitioning columns in the input row recordValue = serializer.serialize(row, subSetOI); } else { - if (lbDirName != null) { - fpaths = valToPaths.get(lbDirName); - if (fpaths == null) { - fpaths = createNewPaths(null, lbDirName); - } - } else { - fpaths = fsp; - } + fpaths = fsp; + recordValue = serializer.serialize(row, inputObjInspectors[0]); // if serializer is ThriftJDBCBinarySerDe, then recordValue is null if the buffer is not full (the size of buffer // is kept track of in the SerDe) @@ -1063,11 +1032,11 @@ assert getConf().getWriteType() != AcidUtils.Operation.DELETE && * @return * @throws HiveException */ - private FSPaths createNewPaths(String dpDir, String lbDir) throws HiveException { + private FSPaths createNewPaths(String dpDir) throws HiveException { FSPaths fsp2 = new FSPaths(specPath, conf.isMmTable()); - fsp2.subdirAfterTxn = combinePathFragments(lbDir, unionPath); + fsp2.subdirAfterTxn = unionPath; fsp2.subdirBeforeTxn = dpDir; - String pathKey = combinePathFragments(dpDir, lbDir); + String pathKey = dpDir; if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("creating new paths {} for {}, childSpec {}: tmpPath {}," + " task path {}", System.identityHashCode(fsp2), pathKey, unionPath, @@ -1084,67 +1053,7 @@ private FSPaths createNewPaths(String dpDir, String lbDir) throws HiveException return fsp2; } - private static String combinePathFragments(String first, String second) { - return first == null ? second : (second == null ? first : first + Path.SEPARATOR + second); - } - - /** - * Generate list bucketing directory name from a row. - * @param row row to process. - * @return directory name. - */ - protected String generateListBucketingDirName(Object row) { - if (!this.isSkewedStoredAsSubDirectories) { - return null; - } - - String lbDirName = null; - List skewedCols = lbCtx.getSkewedColNames(); - List> allSkewedVals = lbCtx.getSkewedColValues(); - Map, String> locationMap = lbCtx.getLbLocationMap(); - - if (row != null) { - List standObjs = new ArrayList(); - List skewedValsCandidate = null; - /* Convert input row to standard objects. */ - ObjectInspectorUtils.copyToStandardObject(standObjs, row, - (StructObjectInspector) inputObjInspectors[0], ObjectInspectorCopyOption.WRITABLE); - - assert (standObjs.size() >= skewedCols.size()) : - "The row has less number of columns than no. of skewed column."; - - skewedValsCandidate = new ArrayList(skewedCols.size()); - for (SkewedColumnPositionPair posPair : lbCtx.getRowSkewedIndex()) { - skewedValsCandidate.add(posPair.getSkewColPosition(), - standObjs.get(posPair.getTblColPosition()).toString()); - } - /* The row matches skewed column names. */ - if (allSkewedVals.contains(skewedValsCandidate)) { - /* matches skewed values. */ - lbDirName = FileUtils.makeListBucketingDirName(skewedCols, skewedValsCandidate); - locationMap.put(skewedValsCandidate, lbDirName); - } else { - lbDirName = createDefaultLbDir(skewedCols, locationMap); - } - } else { - lbDirName = createDefaultLbDir(skewedCols, locationMap); - } - return lbDirName; - } - - private String createDefaultLbDir(List skewedCols, - Map, String> locationMap) { - String lbDirName; - lbDirName = FileUtils.makeDefaultListBucketingDirName(skewedCols, - lbCtx.getDefaultDirName()); - List defaultKey = Lists.newArrayList(lbCtx.getDefaultKey()); - if (!locationMap.containsKey(defaultKey)) { - locationMap.put(defaultKey, lbDirName); - } - return lbDirName; - } - - protected FSPaths getDynOutPaths(List row, String lbDir) throws HiveException { + protected FSPaths getDynOutPaths(List row) throws HiveException { FSPaths fp; @@ -1153,12 +1062,11 @@ protected FSPaths getDynOutPaths(List row, String lbDir) throws HiveExce String pathKey = null; if (dpDir != null) { - String dpAndLbDir = combinePathFragments(dpDir, lbDir); - pathKey = dpAndLbDir; + pathKey = dpDir; if (conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED)) { String buckNum = row.get(row.size() - 1); taskId = Utilities.replaceTaskIdFromFilename(taskId, buckNum); - pathKey = dpAndLbDir + Path.SEPARATOR + taskId; + pathKey = dpDir + Path.SEPARATOR + taskId; } FSPaths fsp2 = valToPaths.get(pathKey); @@ -1202,7 +1110,7 @@ protected FSPaths getDynOutPaths(List row, String lbDir) throws HiveExce prevFsp = null; } - fsp2 = createNewPaths(dpDir, lbDir); + fsp2 = createNewPaths(dpDir); if (prevFsp == null) { prevFsp = fsp2; } @@ -1520,50 +1428,10 @@ private void publishStats() throws HiveException { * This is server side code to create key in order to save statistics to stats database. * Client side will read it via StatsTask.java aggregateStats(). * Client side reads it via db query prefix which is based on partition spec. - * Since store-as-subdir information is not part of partition spec, we have to - * remove store-as-subdir information from variable "keyPrefix" calculation. - * But we have to keep store-as-subdir information in variable "key" calculation - * since each skewed value has a row in stats db and "key" is db key, - * otherwise later value overwrites previous value. - * Performance impact due to string handling is minimum since this method is - * only called once in FileSinkOperator closeOp(). - * For example, - * create table test skewed by (key, value) on (('484','val_484') stored as DIRECTORIES; - * skewedValueDirList contains 2 elements: - * 1. key=484/value=val_484 - * 2. HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * Case #1: Static partition with store-as-sub-dir - * spSpec has SP path - * fspKey has either - * key=484/value=val_484 or - * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * After filter, fspKey is empty, storedAsDirPostFix has either - * key=484/value=val_484 or - * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * so, at the end, "keyPrefix" doesnt have subdir information but "key" has - * Case #2: Dynamic partition with store-as-sub-dir. Assume dp part is hr - * spSpec has SP path - * fspKey has either - * hr=11/key=484/value=val_484 or - * hr=11/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * After filter, fspKey is hr=11, storedAsDirPostFix has either - * key=484/value=val_484 or - * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * so, at the end, "keyPrefix" doesn't have subdir information from skewed but "key" has * - * In a word, fspKey is consists of DP(dynamic partition spec) + LB(list bucketing spec) - * In stats publishing, full partition spec consists of prefix part of stat key - * but list bucketing spec is regarded as a postfix of stat key. So we split it here. + * In a word, fspKey is consists of DP(dynamic partition spec). */ private String[] splitKey(String fspKey) { - if (!fspKey.isEmpty() && isSkewedStoredAsSubDirectories) { - for (String dir : lbCtx.getSkewedValuesDirNames()) { - int index = fspKey.indexOf(dir); - if (index >= 0) { - return new String[] {fspKey.substring(0, index), fspKey.substring(index + 1)}; - } - } - } return new String[] {fspKey, null}; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index f80a945be5..6d393384fa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -378,7 +378,7 @@ public int execute(DriverContext driverContext) { + " into " + tbd.getTable().getTableName()); } db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getLoadFileType(), - work.isSrcLocal(), isSkewedStoredAsDirs(tbd), isFullAcidOp, hasFollowingStatsTask(), + work.isSrcLocal(), isFullAcidOp, hasFollowingStatsTask(), tbd.getWriteId(), tbd.getStmtId(), tbd.isInsertOverwrite()); if (work.getOutputs() != null) { DDLTask.addIfAbsentByName(new WriteEntity(table, @@ -477,11 +477,9 @@ private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, db.loadPartition(tbd.getSourcePath(), db.getTable(tbd.getTable().getTableName()), tbd.getPartitionSpec(), tbd.getLoadFileType(), tbd.getInheritTableSpecs(), - isSkewedStoredAsDirs(tbd), work.isSrcLocal(), - work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && - !tbd.isMmTable(), - hasFollowingStatsTask(), - tbd.getWriteId(), tbd.getStmtId(), tbd.isInsertOverwrite()); + work.isSrcLocal(), + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && !tbd.isMmTable(), + hasFollowingStatsTask(), tbd.getWriteId(), tbd.getStmtId(), tbd.isInsertOverwrite()); Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); // See the comment inside updatePartitionBucketSortColumns. @@ -712,11 +710,6 @@ private void checkFileFormats(Hive db, LoadTableDesc tbd, Table table) } } - private boolean isSkewedStoredAsDirs(LoadTableDesc tbd) { - return (tbd.getLbCtx() == null) ? false : tbd.getLbCtx() - .isSkewedStoredAsDir(); - } - /** * Alters the bucketing and/or sorting columns of the partition provided they meet some * validation criteria, e.g. the number of buckets match the number of files, and the diff --git ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java index 07abd378c5..0f44142848 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java @@ -20,10 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; @@ -38,9 +35,7 @@ import org.apache.hadoop.hive.ql.plan.Explain.Level; import org.apache.hadoop.mapred.InputFormat; -import java.io.IOException; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; @Explain(displayName = "Merge File Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) @@ -142,57 +137,6 @@ public void resolveDynamicPartitionStoredAsSubDirsMerge(HiveConf conf, inputPaths.add(path); } - /** - * alter table ... concatenate - *

- * If it is skewed table, use subdirectories in inputpaths. - */ - public void resolveConcatenateMerge(HiveConf conf) { - isListBucketingAlterTableConcatenate = - ((listBucketingCtx == null) ? false : listBucketingCtx - .isSkewedStoredAsDir()); - LOG.info("isListBucketingAlterTableConcatenate : " + - isListBucketingAlterTableConcatenate); - if (isListBucketingAlterTableConcatenate) { - // use sub-dir as inputpath. - assert ((this.inputPaths != null) && (this.inputPaths.size() == 1)) : - "alter table ... concatenate should only have one" + - " directory inside inputpaths"; - Path dirPath = inputPaths.get(0); - try { - FileSystem inpFs = dirPath.getFileSystem(conf); - List status = HiveStatsUtils.getFileStatusRecurse( - dirPath, listBucketingCtx.getSkewedColNames().size(), inpFs); - List newInputPath = new ArrayList(); - boolean succeed = true; - for (FileStatus s : status) { - if (s.isDir()) { - // Add the lb path to the list of input paths - newInputPath.add(s.getPath()); - } else { - // find file instead of dir. dont change inputpath - succeed = false; - } - } - assert (succeed || ((!succeed) && newInputPath.isEmpty())) : - "This partition has " - + " inconsistent file structure: " - + - "it is stored-as-subdir and expected all files in the same depth" - + " of subdirectories."; - if (succeed) { - inputPaths.clear(); - inputPaths.addAll(newInputPath); - } - } catch (IOException e) { - String msg = - "Fail to get filesystem for directory name : " + dirPath.toUri(); - throw new RuntimeException(msg, e); - } - - } - } - /** * @return the listBucketingCtx */ diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 2ec131e274..d0dfca8c88 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1681,9 +1681,9 @@ public Database getDatabaseCurrent() throws HiveException { * @return Partition object being loaded with data */ public Partition loadPartition(Path loadPath, Table tbl, Map partSpec, - LoadFileType loadFileType, boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir, - boolean isSrcLocal, boolean isAcidIUDoperation, boolean hasFollowingStatsTask, Long writeId, - int stmtId, boolean isInsertOverwrite) throws HiveException { + LoadFileType loadFileType, boolean inheritTableSpecs, boolean isSrcLocal, + boolean isAcidIUDoperation, boolean hasFollowingStatsTask, Long writeId, int stmtId, + boolean isInsertOverwrite) throws HiveException { Path tblDataLocationPath = tbl.getDataLocation(); boolean isMmTableWrite = AcidUtils.isInsertOnlyTable(tbl.getParameters()); assert tbl.getPath() != null : "null==getPath() for " + tbl.getTableName(); @@ -1806,17 +1806,6 @@ public Partition loadPartition(Path loadPath, Table tbl, Map par StatsSetupConst.clearColumnStatsState(newTPart.getParameters()); } - // recreate the partition if it existed before - if (isSkewedStoreAsSubdir) { - org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition(); - SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo(); - /* Construct list bucketing location mappings from sub-directory name. */ - Map, String> skewedColValueLocationMaps = constructListBucketingLocationMap( - newPartPath, skewedInfo); - /* Add list bucketing location mappings. */ - skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps); - newCreatedTpart.getSd().setSkewedInfo(skewedInfo); - } if (!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { StatsSetupConst.setBasicStatsState(newTPart.getParameters(), StatsSetupConst.FALSE); } @@ -2213,7 +2202,7 @@ public Void call() throws Exception { // load the partition Partition newPartition = loadPartition(partPath, tbl, fullPartSpec, loadFileType, - true, numLB > 0, false, isAcid, hasFollowingStatsTask, writeId, stmtId, + true, false, isAcid, hasFollowingStatsTask, writeId, stmtId, isInsertOverwrite); partitionsMap.put(fullPartSpec, newPartition); @@ -2301,8 +2290,6 @@ public Void call() throws Exception { * otherwise add files to table (KEEP_EXISTING, OVERWRITE_EXISTING) * @param isSrcLocal * If the source directory is LOCAL - * @param isSkewedStoreAsSubdir - * if list bucketing enabled * @param hasFollowingStatsTask * if there is any following stats task * @param isAcidIUDoperation true if this is an ACID based Insert [overwrite]/update/delete @@ -2310,12 +2297,11 @@ public Void call() throws Exception { * @param stmtId statement ID of the current load statement */ public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType, boolean isSrcLocal, - boolean isSkewedStoreAsSubdir, boolean isAcidIUDoperation, boolean hasFollowingStatsTask, + boolean isAcidIUDoperation, boolean hasFollowingStatsTask, Long writeId, int stmtId, boolean isInsertOverwrite) throws HiveException { PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin("MoveTask", PerfLogger.LOAD_TABLE); - List newFiles = null; Table tbl = getTable(tableName); assert tbl.getPath() != null : "null==getPath() for " + tbl.getTableName(); @@ -2384,20 +2370,6 @@ public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType StatsSetupConst.clearColumnStatsState(tbl.getParameters()); } - try { - if (isSkewedStoreAsSubdir) { - SkewedInfo skewedInfo = tbl.getSkewedInfo(); - // Construct list bucketing location mappings from sub-directory name. - Map, String> skewedColValueLocationMaps = constructListBucketingLocationMap( - tbl.getPath(), skewedInfo); - // Add list bucketing location mappings. - skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps); - } - } catch (IOException e) { - LOG.error(StringUtils.stringifyException(e)); - throw new HiveException(e); - } - EnvironmentContext environmentContext = null; if (hasFollowingStatsTask) { environmentContext = new EnvironmentContext(); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index 9dbd869d57..f4f76649a2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -552,10 +552,6 @@ public void setLastAccessTime(int lastAccessTime) { tPartition.setLastAccessTime(lastAccessTime); } - public boolean isStoredAsSubDirectories() { - return tPartition.getSd().isStoredAsSubDirectories(); - } - public List> getSkewedColValues(){ return tPartition.getSd().getSkewedInfo().getSkewedColValues(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index f0061c01f6..10ed987608 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -625,14 +625,6 @@ public void setSkewedInfo(SkewedInfo skewedInfo) throws HiveException { tTable.getSd().setSkewedInfo(skewedInfo); } - public boolean isStoredAsSubDirectories() { - return tTable.getSd().isStoredAsSubDirectories(); - } - - public void setStoredAsSubDirectories(boolean storedAsSubDirectories) throws HiveException { - tTable.getSd().setStoredAsSubDirectories(storedAsSubDirectories); - } - private boolean isField(String col) { for (FieldSchema field : getCols()) { if (field.getName().equals(col)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java index 44687ef471..8baac7c8e9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java @@ -382,9 +382,6 @@ private static void getStorageDescriptorInfo(StringBuilder tableInfo, formatOutput("Num Buckets:", String.valueOf(storageDesc.getNumBuckets()), tableInfo); formatOutput("Bucket Columns:", storageDesc.getBucketCols().toString(), tableInfo); formatOutput("Sort Columns:", storageDesc.getSortCols().toString(), tableInfo); - if (storageDesc.isStoredAsSubDirectories()) {// optional parameter - formatOutput("Stored As SubDirectories:", "Yes", tableInfo); - } if (null != storageDesc.getSkewedInfo()) { List skewedColNames = sortedList(storageDesc.getSkewedInfo().getSkewedColNames()); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 605bb09cab..ea6b0b47bd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1621,8 +1621,7 @@ public static MapWork createMergeTask(FileSinkDesc fsInputDesc, Path finalName, ArrayList inputDirstr = new ArrayList(1); // this will be populated by MergeFileWork.resolveDynamicPartitionStoredAsSubDirsMerge // in case of dynamic partitioning and list bucketing - if (!hasDynamicPartitions && - !GenMapRedUtils.isSkewedStoredAsDirs(fsInputDesc)) { + if (!hasDynamicPartitions) { inputDirs.add(inputDir); } inputDirstr.add(inputDir.toString()); @@ -1851,17 +1850,6 @@ private static ConditionalTask createCondTask(HiveConf conf, return cndTsk; } - /** - * check if it is skewed table and stored as dirs. - * - * @param fsInputDesc - * @return - */ - public static boolean isSkewedStoredAsDirs(FileSinkDesc fsInputDesc) { - return (fsInputDesc.getLbCtx() == null) ? false : fsInputDesc.getLbCtx() - .isSkewedStoredAsDir(); - } - public static Task findMoveTaskForFsopOutput( List> mvTasks, Path fsopFinalDir, boolean isMmFsop) { // find the move task diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 596eddedf5..b1ff80abf9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1839,17 +1839,15 @@ public boolean hasTransactionalInQuery() { * @param skewedColNames * @param skewedValues * @param skewedColValueLocationMaps - * @param isStoredAsSubDirectories * @return */ protected ListBucketingCtx constructListBucketingCtx(List skewedColNames, List> skewedValues, Map, String> skewedColValueLocationMaps, - boolean isStoredAsSubDirectories, HiveConf conf) { + HiveConf conf) { ListBucketingCtx lbCtx = new ListBucketingCtx(); lbCtx.setSkewedColNames(skewedColNames); lbCtx.setSkewedColValues(skewedValues); lbCtx.setLbLocationMap(skewedColValueLocationMaps); - lbCtx.setStoredAsSubDirectories(isStoredAsSubDirectories); lbCtx.setDefaultKey(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY); lbCtx.setDefaultDirName(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME); return lbCtx; @@ -1962,22 +1960,6 @@ protected void analyzeDDLSkewedValues(List> skewedValues, ASTNode c } } - /** - * process stored as directories - * - * @param child - * @return - */ - protected boolean analyzeStoredAdDirs(ASTNode child) { - boolean storedAsDirs = false; - if ((child.getChildCount() == 3) - && (((ASTNode) child.getChild(2)).getToken().getType() - == HiveParser.TOK_STOREDASDIRS)) { - storedAsDirs = true; - } - return storedAsDirs; - } - private static boolean getPartExprNodeDesc(ASTNode astNode, HiveConf conf, Map astExprNodeMap) throws SemanticException { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 2dea254c87..f7c2e8ad01 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1503,8 +1503,6 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { Path oldTblPartLoc = null; List cols = null; ListBucketingCtx lbCtx = null; - boolean isListBucketed = false; - List listBucketColNames = null; if (table.isPartitioned()) { Partition part = db.getPartition(table, partSpec, false); @@ -1524,9 +1522,7 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { inputFormatClass = part.getInputFormatClass(); isArchived = ArchiveUtils.isArchived(part); lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(), - part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf); - isListBucketed = part.isStoredAsSubDirectories(); - listBucketColNames = part.getSkewedColNames(); + part.getSkewedColValueLocationMaps(), conf); } else { // input and output are the same oldTblPartLoc = table.getPath(); @@ -1535,9 +1531,7 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { bucketCols = table.getBucketCols(); inputFormatClass = table.getInputFormatClass(); lbCtx = constructListBucketingCtx(table.getSkewedColNames(), table.getSkewedColValues(), - table.getSkewedColValueLocationMaps(), table.isStoredAsSubDirectories(), conf); - isListBucketed = table.isStoredAsSubDirectories(); - listBucketColNames = table.getSkewedColNames(); + table.getSkewedColValueLocationMaps(), conf); } // throw a HiveException for non-rcfile. @@ -1570,14 +1564,6 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { throw new SemanticException(ErrorMsg.TRUNCATE_BUCKETED_COLUMN.getMsg(columnName)); } } - if (isListBucketed) { - for (String listBucketCol : listBucketColNames) { - if (listBucketCol.equalsIgnoreCase(columnName)) { - throw new SemanticException( - ErrorMsg.TRUNCATE_LIST_BUCKETED_COLUMN.getMsg(columnName)); - } - } - } } truncateTblDesc.setColumnIndexes(new ArrayList(columnIndexes)); @@ -2025,7 +2011,7 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, oldTblPartLoc = partPath; lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(), - part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf); + part.getSkewedColValueLocationMaps(), conf); } } else { inputFormatClass = tblObj.getInputFormatClass(); @@ -2036,7 +2022,7 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, newTblPartLoc = tblObj.getPath(); lbCtx = constructListBucketingCtx(tblObj.getSkewedColNames(), tblObj.getSkewedColValues(), - tblObj.getSkewedColValueLocationMaps(), tblObj.isStoredAsSubDirectories(), conf); + tblObj.getSkewedColValueLocationMaps(), conf); } // throw a HiveException for other than rcfile and orcfile. @@ -4074,7 +4060,6 @@ private void analyzeAltertableSkewedby(String[] qualified, ASTNode ast) throws S /* Convert a skewed table to non-skewed table. */ AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, true, new ArrayList(), new ArrayList>()); - alterTblDesc.setStoredAsSubDirectories(false); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc))); } else { @@ -4082,9 +4067,6 @@ private void analyzeAltertableSkewedby(String[] qualified, ASTNode ast) throws S case HiveParser.TOK_TABLESKEWED: handleAlterTableSkewedBy(ast, tableName, tab); break; - case HiveParser.TOK_STOREDASDIRS: - handleAlterTableDisableStoredAsDirs(tableName, tab); - break; default: assert false; } @@ -4092,29 +4074,7 @@ private void analyzeAltertableSkewedby(String[] qualified, ASTNode ast) throws S } /** - * Handle alter table not stored as directories - * - * @param tableName - * @param tab - * @throws SemanticException - */ - private void handleAlterTableDisableStoredAsDirs(String tableName, Table tab) - throws SemanticException { - List skewedColNames = tab.getSkewedColNames(); - List> skewedColValues = tab.getSkewedColValues(); - if ((skewedColNames == null) || (skewedColNames.size() == 0) || (skewedColValues == null) - || (skewedColValues.size() == 0)) { - throw new SemanticException(ErrorMsg.ALTER_TBL_STOREDASDIR_NOT_SKEWED.getMsg(tableName)); - } - AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, false, - skewedColNames, skewedColValues); - alterTblDesc.setStoredAsSubDirectories(false); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - alterTblDesc))); - } - - /** - * Process "alter table skewed by .. on .. stored as directories + * Process "alter table skewed by .. on .. * @param ast * @param tableName * @param tab @@ -4129,13 +4089,9 @@ private void handleAlterTableSkewedBy(ASTNode ast, String tableName, Table tab) skewedColNames = analyzeSkewedTablDDLColNames(skewedColNames, skewedNode); /* skewed value. */ analyzeDDLSkewedValues(skewedValues, skewedNode); - // stored as directories - boolean storedAsDirs = analyzeStoredAdDirs(skewedNode); - AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, false, skewedColNames, skewedValues); - alterTblDesc.setStoredAsSubDirectories(storedAsDirs); /** * Validate information about skewed table */ @@ -4145,51 +4101,6 @@ private void handleAlterTableSkewedBy(ASTNode ast, String tableName, Table tab) alterTblDesc))); } - /** - * Analyze skewed column names - * - * @param skewedColNames - * @param child - * @return - * @throws SemanticException - */ - private List analyzeAlterTableSkewedColNames(List skewedColNames, - ASTNode child) throws SemanticException { - Tree nNode = child.getChild(0); - if (nNode == null) { - throw new SemanticException(ErrorMsg.SKEWED_TABLE_NO_COLUMN_NAME.getMsg()); - } else { - ASTNode nAstNode = (ASTNode) nNode; - if (nAstNode.getToken().getType() != HiveParser.TOK_TABCOLNAME) { - throw new SemanticException(ErrorMsg.SKEWED_TABLE_NO_COLUMN_NAME.getMsg()); - } else { - skewedColNames = getColumnNames(nAstNode); - } - } - return skewedColNames; - } - - /** - * Given a ASTNode, return list of values. - * - * use case: - * create table xyz list bucketed (col1) with skew (1,2,5) - * AST Node is for (1,2,5) - * - * @param ast - * @return - */ - private List getColumnValues(ASTNode ast) { - List colList = new ArrayList(); - int numCh = ast.getChildCount(); - for (int i = 0; i < numCh; i++) { - ASTNode child = (ASTNode) ast.getChild(i); - colList.add(stripQuotes(child.getText()).toLowerCase()); - } - return colList; - } - - /** * Analyze alter table's skewed location * diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 324c80487e..e3769620b3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -340,7 +340,6 @@ TOK_TABCOLVALUES; TOK_SKEWED_LOCATIONS; TOK_SKEWED_LOCATION_LIST; TOK_SKEWED_LOCATION_MAP; -TOK_STOREDASDIRS; TOK_PARTITIONINGSPEC; TOK_PTBLFUNCTION; TOK_WINDOWDEF; @@ -997,13 +996,6 @@ rewriteDisabled -> ^(TOK_REWRITE_DISABLED) ; -storedAsDirs -@init { pushMsg("stored as directories", state); } -@after { popMsg(state); } - : KW_STORED KW_AS KW_DIRECTORIES - -> ^(TOK_STOREDASDIRS) - ; - orReplace @init { pushMsg("or replace clause", state); } @after { popMsg(state); } @@ -1433,9 +1425,6 @@ alterStatementSuffixSkewedby | KW_NOT KW_SKEWED ->^(TOK_ALTERTABLE_SKEWED) - | - KW_NOT storedAsDirs - ->^(TOK_ALTERTABLE_SKEWED storedAsDirs) ; alterStatementSuffixExchangePartition @@ -1994,8 +1983,8 @@ tableSkewed @init { pushMsg("table skewed specification", state); } @after { popMsg(state); } : - KW_SKEWED KW_BY LPAREN skewedCols=columnNameList RPAREN KW_ON LPAREN (skewedValues=skewedValueElement) RPAREN ((storedAsDirs) => storedAsDirs)? - -> ^(TOK_TABLESKEWED $skewedCols $skewedValues storedAsDirs?) + KW_SKEWED KW_BY LPAREN skewedCols=columnNameList RPAREN KW_ON LPAREN (skewedValues=skewedValueElement) RPAREN + -> ^(TOK_TABLESKEWED $skewedCols $skewedValues) ; rowFormat diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index cbacd05498..83c70c0929 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -316,9 +316,6 @@ private void analyzeLoad(ASTNode ast) throws SemanticException { throw new SemanticException(ErrorMsg.LOAD_INTO_NON_NATIVE.getMsg()); } - if(ts.tableHandle.isStoredAsSubDirectories()) { - throw new SemanticException(ErrorMsg.LOAD_INTO_STORED_AS_DIR.getMsg()); - } List parts = ts.tableHandle.getPartitionKeys(); if ((parts != null && parts.size() > 0) && (ts.partSpec == null || ts.partSpec.size() == 0)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index c2bceddd7c..221f8cf11b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7285,9 +7285,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) currentTableId = destTableId; destTableId++; - lbCtx = constructListBucketingCtx(dest_tab.getSkewedColNames(), - dest_tab.getSkewedColValues(), dest_tab.getSkewedColValueLocationMaps(), - dest_tab.isStoredAsSubDirectories(), conf); + lbCtx = constructListBucketingCtx(dest_tab.getSkewedColNames(), dest_tab.getSkewedColValues(), + dest_tab.getSkewedColValueLocationMaps(), conf); // Create the work for moving the table // NOTE: specify Dynamic partitions in dest_tab for WriteEntity @@ -7387,8 +7386,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) destTableId++; lbCtx = constructListBucketingCtx(dest_part.getSkewedColNames(), - dest_part.getSkewedColValues(), dest_part.getSkewedColValueLocationMaps(), - dest_part.isStoredAsSubDirectories(), conf); + dest_part.getSkewedColValues(), dest_part.getSkewedColValueLocationMaps(), conf); AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID; if (destTableIsFullAcid) { acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest); @@ -7784,7 +7782,6 @@ private FileSinkDesc createFileSinkDesc(String dest, TableDesc table_desc, /* Set List Bucketing context. */ if (lbCtx != null) { lbCtx.processRowSkewedIndex(fsRS); - lbCtx.calculateSkewedValueSubDirList(); } fileSinkDesc.setLbCtx(lbCtx); @@ -12962,7 +12959,6 @@ ASTNode analyzeCreateTable( List skewedColNames = new ArrayList(); List> skewedValues = new ArrayList>(); Map, String> listBucketColValuesMapping = new HashMap, String>(); - boolean storedAsDirs = false; boolean isUserStorageFormat = false; RowFormatParams rowFormatParams = new RowFormatParams(); @@ -13092,9 +13088,6 @@ ASTNode analyzeCreateTable( skewedColNames = analyzeSkewedTablDDLColNames(skewedColNames, child); // skewed value analyzeDDLSkewedValues(skewedValues, child); - // stored as directories - storedAsDirs = analyzeStoredAdDirs(child); - break; default: throw new AssertionError("Unknown token: " + child.getToken()); @@ -13171,7 +13164,6 @@ ASTNode analyzeCreateTable( storageFormat.getStorageHandler(), storageFormat.getSerdeProps(), tblProps, ifNotExists, skewedColNames, skewedValues, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints); - crtTblDesc.setStoredAsSubDirectories(storedAsDirs); crtTblDesc.setNullFormat(rowFormatParams.nullFormat); crtTblDesc.validate(conf); @@ -13274,7 +13266,6 @@ ASTNode analyzeCreateTable( skewedColNames, skewedValues, true, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints); tableDesc.setMaterialization(isMaterialization); - tableDesc.setStoredAsSubDirectories(storedAsDirs); tableDesc.setNullFormat(rowFormatParams.nullFormat); qb.setTableDesc(tableDesc); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java index 7925151012..c3bc1d5fda 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -1118,7 +1118,6 @@ private boolean handleCardinalityViolation(StringBuilder rewrittenQueryStr, ASTN table.setDataLocation(Warehouse.getDnsPath(new Path(SessionState.get().getTempTableSpace(), tableName), conf)); table.getTTable().setTemporary(true); - table.setStoredAsSubDirectories(false); table.setInputFormatClass(format.getInputFormat()); table.setOutputFormatClass(format.getOutputFormat()); db.createTable(table, true); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java index 0b04c0ce85..ba03e88630 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java @@ -121,7 +121,6 @@ ProtectModeType protectModeType; Map, String> skewedLocations; boolean isTurnOffSkewed = false; - boolean isStoredAsSubDirectories = false; List skewedColNames; List> skewedColValues; Table tableForSkewedColValidation; @@ -888,20 +887,6 @@ public void setTable(Table table) { this.tableForSkewedColValidation = table; } - /** - * @return the isStoredAsSubDirectories - */ - public boolean isStoredAsSubDirectories() { - return isStoredAsSubDirectories; - } - - /** - * @param isStoredAsSubDirectories the isStoredAsSubDirectories to set - */ - public void setStoredAsSubDirectories(boolean isStoredAsSubDirectories) { - this.isStoredAsSubDirectories = isStoredAsSubDirectories; - } - /** * @param isDropIfExists the isDropIfExists to set */ diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java index 871844b30d..8250f33af0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java @@ -92,7 +92,6 @@ boolean ifNotExists; List skewedColNames; List> skewedColValues; - boolean isStoredAsSubDirectories = false; boolean isTemporary = false; private boolean isMaterialization = false; private boolean replaceMode = false; @@ -605,20 +604,6 @@ public void validate(HiveConf conf) this.getSkewedColValues()); } - /** - * @return the isStoredAsSubDirectories - */ - public boolean isStoredAsSubDirectories() { - return isStoredAsSubDirectories; - } - - /** - * @param isStoredAsSubDirectories the isStoredAsSubDirectories to set - */ - public void setStoredAsSubDirectories(boolean isStoredAsSubDirectories) { - this.isStoredAsSubDirectories = isStoredAsSubDirectories; - } - /** * @return the nullFormat */ @@ -806,8 +791,6 @@ public Table toTable(HiveConf conf) throws HiveException { tbl.getTTable().setTemporary(isTemporary()); - tbl.setStoredAsSubDirectories(isStoredAsSubDirectories()); - tbl.setInputFormatClass(getInputFormat()); tbl.setOutputFormatClass(getOutputFormat()); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java index ef7325fe2c..ed80240c65 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java @@ -85,7 +85,6 @@ public ImportTableDesc(String dbName, Table table) throws Exception { null, null, null); - this.createTblDesc.setStoredAsSubDirectories(table.getSd().isStoredAsSubDirectories()); break; case VIEW: String[] qualViewName = { dbName, table.getTableName() }; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java index 5f8cf54d57..c326b590df 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java @@ -41,14 +41,11 @@ private List> skewedColValues; private Map, String> lbLocationMap; private List rowSkewedIndex; - private boolean isStoredAsSubDirectories; private String defaultKey; private String defaultDirName; - private List skewedValuesDirNames; public ListBucketingCtx() { rowSkewedIndex = new ArrayList(); - skewedValuesDirNames = new ArrayList(); } /** @@ -118,41 +115,6 @@ public void processRowSkewedIndex(RowSchema rowSch) { } } - /** - * Calculate skewed value subdirectory directory which is used in - * FileSinkOperator.java createKeyForStatsPublisher() - * For example, create table test skewed by (key, value) on (('484','val_484') - * stored as DIRECTORIES; - * after the method, skewedValuesDirNames will contain 2 elements: - * key=484/value=val_484 - * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - */ - public void calculateSkewedValueSubDirList() { - if (isSkewedStoredAsDir()) { - for (List value : this.skewedColValues) { - skewedValuesDirNames.add(FileUtils.makeListBucketingDirName(this.skewedColNames, value)); - } - // creat default dir - skewedValuesDirNames.add(FileUtils.makeDefaultListBucketingDirName( - this.skewedColNames, - ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)); - } - } - - /** - * @return the isStoredAsSubDirectories - */ - public boolean isStoredAsSubDirectories() { - return isStoredAsSubDirectories; - } - - /** - * @param isStoredAsSubDirectories the isStoredAsSubDirectories to set - */ - public void setStoredAsSubDirectories(boolean isStoredAsSubDirectories) { - this.isStoredAsSubDirectories = isStoredAsSubDirectories; - } - /** * @return the defaultKey */ @@ -181,46 +143,16 @@ public void setDefaultDirName(String defaultDirName) { this.defaultDirName = defaultDirName; } - /** - * check if list bucketing is enabled. - * - * @param ctx - * @return - */ - public boolean isSkewedStoredAsDir() { - return (this.getSkewedColNames() != null) - && (this.getSkewedColNames().size() > 0) - && (this.getSkewedColValues() != null) - && (this.getSkewedColValues().size() > 0) - && (this.isStoredAsSubDirectories()); - } - /** * Calculate list bucketing level. * - * 0: not list bucketing - * int: no. of skewed columns + * no list bucketing, it's been removed with HIVE-17852 * * @param ctx * @return */ - public int calculateListBucketingLevel() { - int lbLevel = isSkewedStoredAsDir() ? this.getSkewedColNames().size() : 0; - return lbLevel; - } - - /** - * @return the skewedValuesDirNames - */ - public List getSkewedValuesDirNames() { - return skewedValuesDirNames; - } - - /** - * @param skewedValuesDirNames the skewedValuesDirNames to set - */ - public void setSkewedValuesDirNames(List skewedValuesDirNames) { - this.skewedValuesDirNames = skewedValuesDirNames; + public int calculateListBucketingLevel() { + return 0; } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/SkewedColumnPositionPair.java ql/src/java/org/apache/hadoop/hive/ql/plan/SkewedColumnPositionPair.java index 8bb40abc8d..eb2ba492ad 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/SkewedColumnPositionPair.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/SkewedColumnPositionPair.java @@ -25,14 +25,11 @@ * create table list_bucketing_static_part (key String, value String) * partitioned by (ds String, hr String) * skewed by (value) on ('val_466','val_287','val_82') - * stored as DIRECTORIES * STORED AS RCFILE; * * Skewed column is "value". * 1. It's position in table column is 1. * 2. It's position in skewed column list is 0. - * - * This information will be used in {@FileSinkOperator} generateListBucketingDirName */ public class SkewedColumnPositionPair { private int tblColPosition; diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index 60447192b1..aa75195c05 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -44,24 +44,20 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.StringableMap; import org.apache.hadoop.hive.common.ValidCompactorWriteIdList; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.CompactionType; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.txn.CompactionInfo; import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.DriverUtils; import org.apache.hadoop.hive.ql.QueryState; @@ -76,7 +72,6 @@ import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; -import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -436,24 +431,6 @@ private String buildMmCompactionCtQuery( query.append("INTO ").append(t.getSd().getNumBuckets()).append(" BUCKETS"); } - // Stored as directories. We don't care about the skew otherwise. - if (t.getSd().isStoredAsSubDirectories()) { - SkewedInfo skewedInfo = t.getSd().getSkewedInfo(); - if (skewedInfo != null && !skewedInfo.getSkewedColNames().isEmpty()) { - query.append(" SKEWED BY (").append( - StringUtils.join(", ", skewedInfo.getSkewedColNames())).append(") ON "); - isFirst = true; - for (List colValues : skewedInfo.getSkewedColValues()) { - if (!isFirst) { - query.append(", "); - } - isFirst = false; - query.append("('").append(StringUtils.join("','", colValues)).append("')"); - } - query.append(") STORED AS DIRECTORIES"); - } - } - SerDeInfo serdeInfo = sd.getSerdeInfo(); Map serdeParams = serdeInfo.getParameters(); query.append(" ROW FORMAT SERDE '").append(HiveStringUtils.escapeHiveCommand( diff --git ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedUtils.java ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedUtils.java index d9536eb24a..d06b78468e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedUtils.java @@ -99,7 +99,7 @@ public static boolean isAvroTableWithExternalSchema(org.apache.hadoop.hive.metas } public static boolean isListBucketedTable(org.apache.hadoop.hive.metastore.api.Table table) { - return table.getSd().isStoredAsSubDirectories(); + return table.getSd().getNumBuckets() > 0; } private static String createValidationError(org.apache.hadoop.hive.metastore.api.Table table, String message) { diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index e108684660..bf366d9e52 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -142,7 +142,7 @@ db.createTable(src, cols, null, TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class); db.loadTable(hadoopDataFile[i], src, LoadFileType.KEEP_EXISTING, - true, false, false, false, null, 0, false); + true, false, false, null, 0, false); i++; } diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index a24b6423ba..bd0a01bcf0 100755 --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -165,7 +165,6 @@ public void testTable() throws Throwable { tbl.setSerdeParam(serdeConstants.FIELD_DELIM, "1"); tbl.setSerializationLib(LazySimpleSerDe.class.getName()); - tbl.setStoredAsSubDirectories(false); tbl.setRewriteEnabled(false); @@ -226,7 +225,6 @@ public void testThriftTable() throws Throwable { tbl.setSerdeParam(serdeConstants.SERIALIZATION_CLASS, Complex.class.getName()); tbl.setSerdeParam(serdeConstants.SERIALIZATION_FORMAT, TBinaryProtocol.class .getName()); - tbl.setStoredAsSubDirectories(false); tbl.setRewriteEnabled(false); diff --git ql/src/test/queries/clientnegative/load_stored_as_dirs.q ql/src/test/queries/clientnegative/load_stored_as_dirs.q deleted file mode 100644 index 69e23099b9..0000000000 --- ql/src/test/queries/clientnegative/load_stored_as_dirs.q +++ /dev/null @@ -1,5 +0,0 @@ --- Load data can't work with table with stored as directories -CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES; - -LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE stored_as_dirs_multiple; diff --git ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q deleted file mode 100644 index 70cbce865a..0000000000 --- ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q +++ /dev/null @@ -1,14 +0,0 @@ ---! qt:dataset:src -set mapred.input.dir.recursive=true; - --- Tests truncating a column on which a table is list bucketed - -CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE; - -ALTER TABLE test_tab -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES; - -INSERT OVERWRITE TABLE test_tab SELECT * FROM src; - -TRUNCATE TABLE test_tab COLUMNS (key); diff --git ql/src/test/queries/clientpositive/create_alter_list_bucketing_table1.q ql/src/test/queries/clientpositive/create_alter_list_bucketing_table1.q deleted file mode 100644 index bf89e8feb7..0000000000 --- ql/src/test/queries/clientpositive/create_alter_list_bucketing_table1.q +++ /dev/null @@ -1,38 +0,0 @@ --- Test stored as directories --- it covers a few cases - --- 1. create a table with stored as directories -CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES; -describe formatted stored_as_dirs_multiple; - --- 2. turn off stored as directories but table is still a skewed table -alter table stored_as_dirs_multiple not stored as DIRECTORIES; -describe formatted stored_as_dirs_multiple; - --- 3. turn off skewed -alter table stored_as_dirs_multiple not skewed; -describe formatted stored_as_dirs_multiple; - --- 4. alter a table to stored as directories -CREATE TABLE stored_as_dirs_single (key STRING, value STRING); -alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES; -describe formatted stored_as_dirs_single; - --- 5. turn off skewed should turn off stored as directories too -alter table stored_as_dirs_single not skewed; -describe formatted stored_as_dirs_single; - --- 6. turn on stored as directories again -alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES; -describe formatted stored_as_dirs_single; - --- 7. create table like -create table stored_as_dirs_single_like like stored_as_dirs_single; -describe formatted stored_as_dirs_single_like; - --- cleanup -drop table stored_as_dirs_single; -drop table stored_as_dirs_multiple; diff --git ql/src/test/queries/clientpositive/infer_bucket_sort_list_bucket.q ql/src/test/queries/clientpositive/infer_bucket_sort_list_bucket.q index c9fd7121ab..e8af47245e 100644 --- ql/src/test/queries/clientpositive/infer_bucket_sort_list_bucket.q +++ ql/src/test/queries/clientpositive/infer_bucket_sort_list_bucket.q @@ -10,8 +10,7 @@ set mapred.input.dir.recursive=true; -- create a skewed table CREATE TABLE list_bucketing_table (key STRING, value STRING) PARTITIONED BY (part STRING) -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES; +SKEWED BY (key) ON ("484"); -- Tests group by, the output should neither be bucketed nor sorted @@ -23,8 +22,7 @@ DESC FORMATTED list_bucketing_table PARTITION (part = '1'); -- create a table skewed on a key which doesnt exist in the data CREATE TABLE list_bucketing_table2 (key STRING, value STRING) PARTITIONED BY (part STRING) -SKEWED BY (key) ON ("abc") -STORED AS DIRECTORIES; +SKEWED BY (key) ON ("abc"); -- should not be bucketed or sorted INSERT OVERWRITE TABLE list_bucketing_table2 PARTITION (part = '1') diff --git ql/src/test/queries/clientpositive/lb_fs_stats.q ql/src/test/queries/clientpositive/lb_fs_stats.q index 3bc3924e8c..85e6133977 100644 --- ql/src/test/queries/clientpositive/lb_fs_stats.q +++ ql/src/test/queries/clientpositive/lb_fs_stats.q @@ -10,7 +10,7 @@ set hive.stats.dbclass=fs; CREATE TABLE test_tab_n0 (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE; -ALTER TABLE test_tab_n0 SKEWED BY (key) ON ("484") STORED AS DIRECTORIES; +ALTER TABLE test_tab_n0 SKEWED BY (key) ON ("484"); INSERT OVERWRITE TABLE test_tab_n0 PARTITION (part = '1') SELECT * FROM src; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_1.q ql/src/test/queries/clientpositive/list_bucket_dml_1.q index 40d5393b0c..04ad05ee59 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_1.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_1.q @@ -13,9 +13,7 @@ set mapred.input.dir.recursive=true; -- create a skewed table create table list_bucketing_dynamic_part_n0 (key String, value String) partitioned by (ds String, hr String) -skewed by (key) on ("484") -stored as DIRECTORIES -; +skewed by (key) on ("484"); -- list bucketing DML explain extended diff --git ql/src/test/queries/clientpositive/list_bucket_dml_10.q ql/src/test/queries/clientpositive/list_bucket_dml_10.q index 60bad7fac2..f3437c4f67 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_10.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_10.q @@ -18,7 +18,6 @@ set mapred.input.dir.recursive=true; create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_11.q ql/src/test/queries/clientpositive/list_bucket_dml_11.q index 7cac6d7374..990c6a5f13 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_11.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_11.q @@ -13,7 +13,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_static_part_n3 (key String, value String) partitioned by (ds String, hr String) skewed by (value) on ('val_466','val_287','val_82') - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_12.q ql/src/test/queries/clientpositive/list_bucket_dml_12.q index be02096641..b0f5e561c4 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_12.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_12.q @@ -12,7 +12,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_mul_col_n0 (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML diff --git ql/src/test/queries/clientpositive/list_bucket_dml_13.q ql/src/test/queries/clientpositive/list_bucket_dml_13.q index 77b010d97b..2a1b163050 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_13.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_13.q @@ -12,7 +12,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML diff --git ql/src/test/queries/clientpositive/list_bucket_dml_14.q ql/src/test/queries/clientpositive/list_bucket_dml_14.q index f640853f20..be6b5d6477 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_14.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_14.q @@ -11,9 +11,7 @@ set mapred.input.dir.recursive=true; -- create a skewed table create table list_bucketing (key String, value String) -skewed by (key) on ("484") -stored as DIRECTORIES -; +skewed by (key) on ("484"); -- list bucketing DML explain extended diff --git ql/src/test/queries/clientpositive/list_bucket_dml_2.q ql/src/test/queries/clientpositive/list_bucket_dml_2.q index 6a468287c0..9b70519988 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_2.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_2.q @@ -26,7 +26,6 @@ set hive.stats.reliable=true; create table list_bucketing_static_part_n4 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_3.q ql/src/test/queries/clientpositive/list_bucket_dml_3.q index 0c1e43a6d4..861744bf0b 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_3.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_3.q @@ -11,7 +11,7 @@ set mapred.input.dir.recursive=true; -- create a skewed table -create table list_bucketing_static_part_n1 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") stored as DIRECTORIES; +create table list_bucketing_static_part_n1 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484"); -- list bucketing DML explain extended diff --git ql/src/test/queries/clientpositive/list_bucket_dml_4.q ql/src/test/queries/clientpositive/list_bucket_dml_4.q index f14efe266d..593c5b6935 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_4.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_4.q @@ -29,7 +29,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_static_part_n2 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_5.q ql/src/test/queries/clientpositive/list_bucket_dml_5.q index 996fa1df97..b215704547 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_5.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_5.q @@ -14,8 +14,7 @@ set mapred.input.dir.recursive=true; -- create a skewed table create table list_bucketing_dynamic_part_n1 (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) -stored as DIRECTORIES; +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')); -- list bucketing DML explain extended diff --git ql/src/test/queries/clientpositive/list_bucket_dml_6.q ql/src/test/queries/clientpositive/list_bucket_dml_6.q index e761c86388..7b30181a91 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_6.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_6.q @@ -52,7 +52,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_dynamic_part_n3 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_7.q ql/src/test/queries/clientpositive/list_bucket_dml_7.q index ce87e4f1a8..8968542313 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_7.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_7.q @@ -27,7 +27,6 @@ set hive.merge.rcfile.block.level=true; CREATE TABLE list_bucketing_dynamic_part (key String, value STRING) PARTITIONED BY (ds string, hr string) skewed by (key) on ('484') - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_8.q ql/src/test/queries/clientpositive/list_bucket_dml_8.q index ca19d7928e..5ee38ee693 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_8.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_8.q @@ -53,7 +53,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_dynamic_part_n2 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_9.q ql/src/test/queries/clientpositive/list_bucket_dml_9.q index 70ef57b1be..23afd576dd 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_9.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_9.q @@ -29,7 +29,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_static_part_n0 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','103') - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q index b8bf94482a..f74190e617 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q +++ ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q @@ -21,8 +21,7 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; -- create a skewed table create table fact_daily (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES; +skewed by (key, value) on (('484','val_484'),('238','val_238')); insert overwrite table fact_daily partition (ds = '1', hr = '4') select key, value from src; diff --git ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q index f9335456ad..ca28b3a30b 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q +++ ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q @@ -19,8 +19,7 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; -- create a skewed table create table fact_daily_n2 (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES; +skewed by (key, value) on (('484','val_484'),('238','val_238')); insert overwrite table fact_daily_n2 partition (ds = '1', hr = '4') select key, value from src; diff --git ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q index d5f6a26b10..afea218db0 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q +++ ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q @@ -27,13 +27,13 @@ select key, value from src; describe formatted fact_daily_n3 PARTITION (ds = '1', hr='1'); -- partition. skewed value is 484/238 -alter table fact_daily_n3 skewed by (key, value) on (('484','val_484'),('238','val_238')) stored as DIRECTORIES; +alter table fact_daily_n3 skewed by (key, value) on (('484','val_484'),('238','val_238')); insert overwrite table fact_daily_n3 partition (ds = '1', hr = '2') select key, value from src; describe formatted fact_daily_n3 PARTITION (ds = '1', hr='2'); -- another partition. skewed value is 327 -alter table fact_daily_n3 skewed by (key, value) on (('327','val_327')) stored as DIRECTORIES; +alter table fact_daily_n3 skewed by (key, value) on (('327','val_327')); insert overwrite table fact_daily_n3 partition (ds = '1', hr = '3') select key, value from src; describe formatted fact_daily_n3 PARTITION (ds = '1', hr='3'); diff --git ql/src/test/queries/clientpositive/mm_all.q ql/src/test/queries/clientpositive/mm_all.q index 61dd3e7475..115f0c7444 100644 --- ql/src/test/queries/clientpositive/mm_all.q +++ ql/src/test/queries/clientpositive/mm_all.q @@ -124,7 +124,7 @@ drop table partunion_mm; set mapreduce.input.fileinputformat.input.dir.recursive=true; create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only"); +tblproperties ("transactional"="true", "transactional_properties"="insert_only"); insert into table skew_mm select key, key, key from intermediate_n0; @@ -134,7 +134,7 @@ drop table skew_mm; create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only"); +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only"); insert into table skew_dp_union_mm partition (k3) select key as i, key as j, key as k, key as l from intermediate_n0 diff --git ql/src/test/queries/clientpositive/show_create_table_db_table.q ql/src/test/queries/clientpositive/show_create_table_db_table.q index 35ac5bf73e..f98ea8abc9 100644 --- ql/src/test/queries/clientpositive/show_create_table_db_table.q +++ ql/src/test/queries/clientpositive/show_create_table_db_table.q @@ -4,7 +4,7 @@ CREATE DATABASE tmp_feng comment 'for show create table test'; SHOW DATABASES; CREATE TABLE tmp_feng.tmp_showcrt1(key string, value int); CREATE TABLE tmp_feng.tmp_showcrt2(key string, value int) skewed by (key) on ('1','2'); -CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') stored as directories; +CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2'); USE default; SHOW CREATE TABLE tmp_feng.tmp_showcrt1; SHOW CREATE TABLE tmp_feng.tmp_showcrt2; diff --git ql/src/test/queries/clientpositive/stats_list_bucket.q ql/src/test/queries/clientpositive/stats_list_bucket.q index bbb4206db4..4f426362ae 100644 --- ql/src/test/queries/clientpositive/stats_list_bucket.q +++ ql/src/test/queries/clientpositive/stats_list_bucket.q @@ -9,7 +9,6 @@ create table stats_list_bucket ( c2 string ) partitioned by (ds string, hr string) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile; @@ -29,7 +28,6 @@ create table stats_list_bucket_1 ( c2 string ) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile; insert overwrite table stats_list_bucket_1 diff --git ql/src/test/queries/clientpositive/truncate_column_list_bucket.q ql/src/test/queries/clientpositive/truncate_column_list_bucket.q index eb08af10e6..876e092305 100644 --- ql/src/test/queries/clientpositive/truncate_column_list_bucket.q +++ ql/src/test/queries/clientpositive/truncate_column_list_bucket.q @@ -12,8 +12,7 @@ set mapred.input.dir.recursive=true; CREATE TABLE test_tab_n3 (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE; ALTER TABLE test_tab_n3 -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES; +SKEWED BY (key) ON ("484"); INSERT OVERWRITE TABLE test_tab_n3 PARTITION (part = '1') SELECT * FROM src; diff --git ql/src/test/results/clientnegative/load_stored_as_dirs.q.out ql/src/test/results/clientnegative/load_stored_as_dirs.q.out deleted file mode 100644 index bcc50e59b8..0000000000 --- ql/src/test/results/clientnegative/load_stored_as_dirs.q.out +++ /dev/null @@ -1,11 +0,0 @@ -PREHOOK: query: CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@stored_as_dirs_multiple -FAILED: SemanticException [Error 10195]: A stored-as-directories table cannot be used as target for LOAD diff --git ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out deleted file mode 100644 index 3b6ddde221..0000000000 --- ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out +++ /dev/null @@ -1,31 +0,0 @@ -PREHOOK: query: CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test_tab -POSTHOOK: query: CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test_tab -PREHOOK: query: ALTER TABLE test_tab -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@test_tab -PREHOOK: Output: default@test_tab -POSTHOOK: query: ALTER TABLE test_tab -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@test_tab -POSTHOOK: Output: default@test_tab -PREHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@test_tab -POSTHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@test_tab -POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: A column on which a partition/table is list bucketed cannot be truncated. key diff --git ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out deleted file mode 100644 index 593ae83899..0000000000 --- ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out +++ /dev/null @@ -1,352 +0,0 @@ -PREHOOK: query: CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@stored_as_dirs_multiple -PREHOOK: query: describe formatted stored_as_dirs_multiple -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: query: describe formatted stored_as_dirs_multiple -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_multiple -# col_name data_type comment -col1 string -col2 int -col3 string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}} - bucketing_version 2 - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Stored As SubDirectories: Yes -Skewed Columns: [col1, col2] -Skewed Values: [[s1, 1], [s13, 13], [s3, 3], [s78, 78]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: alter table stored_as_dirs_multiple not stored as DIRECTORIES -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_multiple -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: alter table stored_as_dirs_multiple not stored as DIRECTORIES -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: Output: default@stored_as_dirs_multiple -PREHOOK: query: describe formatted stored_as_dirs_multiple -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: query: describe formatted stored_as_dirs_multiple -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_multiple -# col_name data_type comment -col1 string -col2 int -col3 string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Skewed Columns: [col1, col2] -Skewed Values: [[s1, 1], [s13, 13], [s3, 3], [s78, 78]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: alter table stored_as_dirs_multiple not skewed -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_multiple -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: alter table stored_as_dirs_multiple not skewed -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: Output: default@stored_as_dirs_multiple -PREHOOK: query: describe formatted stored_as_dirs_multiple -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: query: describe formatted stored_as_dirs_multiple -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_multiple -# col_name data_type comment -col1 string -col2 int -col3 string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: CREATE TABLE stored_as_dirs_single (key STRING, value STRING) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: CREATE TABLE stored_as_dirs_single (key STRING, value STRING) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_single -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_single -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: describe formatted stored_as_dirs_single -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_single -POSTHOOK: query: describe formatted stored_as_dirs_single -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_single -# col_name data_type comment -key string -value string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Stored As SubDirectories: Yes -Skewed Columns: [key] -Skewed Values: [[1], [5], [6]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: alter table stored_as_dirs_single not skewed -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_single -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: alter table stored_as_dirs_single not skewed -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_single -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: describe formatted stored_as_dirs_single -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_single -POSTHOOK: query: describe formatted stored_as_dirs_single -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_single -# col_name data_type comment -key string -value string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_single -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_single -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: describe formatted stored_as_dirs_single -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_single -POSTHOOK: query: describe formatted stored_as_dirs_single -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_single -# col_name data_type comment -key string -value string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Stored As SubDirectories: Yes -Skewed Columns: [key] -Skewed Values: [[1], [5], [6]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: create table stored_as_dirs_single_like like stored_as_dirs_single -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@stored_as_dirs_single_like -POSTHOOK: query: create table stored_as_dirs_single_like like stored_as_dirs_single -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@stored_as_dirs_single_like -PREHOOK: query: describe formatted stored_as_dirs_single_like -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_single_like -POSTHOOK: query: describe formatted stored_as_dirs_single_like -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_single_like -# col_name data_type comment -key string -value string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Stored As SubDirectories: Yes -Skewed Columns: [key] -Skewed Values: [[1], [5], [6]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: drop table stored_as_dirs_single -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@stored_as_dirs_single -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: drop table stored_as_dirs_single -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@stored_as_dirs_single -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: drop table stored_as_dirs_multiple -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@stored_as_dirs_multiple -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: drop table stored_as_dirs_multiple -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: Output: default@stored_as_dirs_multiple diff --git ql/src/test/results/clientpositive/infer_bucket_sort_list_bucket.q.out ql/src/test/results/clientpositive/infer_bucket_sort_list_bucket.q.out index c701de9347..7cf9943263 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_list_bucket.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_list_bucket.q.out @@ -1,14 +1,12 @@ PREHOOK: query: CREATE TABLE list_bucketing_table (key STRING, value STRING) PARTITIONED BY (part STRING) SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_table POSTHOOK: query: CREATE TABLE list_bucketing_table (key STRING, value STRING) PARTITIONED BY (part STRING) SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_table @@ -45,7 +43,7 @@ Table: list_bucketing_table #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 309 rawDataSize 1482 totalSize 1791 @@ -59,24 +57,19 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_table/part=1/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: CREATE TABLE list_bucketing_table2 (key STRING, value STRING) PARTITIONED BY (part STRING) SKEWED BY (key) ON ("abc") -STORED AS DIRECTORIES PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_table2 POSTHOOK: query: CREATE TABLE list_bucketing_table2 (key STRING, value STRING) PARTITIONED BY (part STRING) SKEWED BY (key) ON ("abc") -STORED AS DIRECTORIES POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_table2 @@ -127,7 +120,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[abc]] Storage Desc Params: diff --git ql/src/test/results/clientpositive/lb_fs_stats.q.out ql/src/test/results/clientpositive/lb_fs_stats.q.out index ac4f7b55b4..8738765826 100644 --- ql/src/test/results/clientpositive/lb_fs_stats.q.out +++ ql/src/test/results/clientpositive/lb_fs_stats.q.out @@ -6,11 +6,11 @@ POSTHOOK: query: CREATE TABLE test_tab_n0 (key STRING, value STRING) PARTITIONED POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@test_tab_n0 -PREHOOK: query: ALTER TABLE test_tab_n0 SKEWED BY (key) ON ("484") STORED AS DIRECTORIES +PREHOOK: query: ALTER TABLE test_tab_n0 SKEWED BY (key) ON ("484") PREHOOK: type: ALTERTABLE_SKEWED PREHOOK: Input: default@test_tab_n0 PREHOOK: Output: default@test_tab_n0 -POSTHOOK: query: ALTER TABLE test_tab_n0 SKEWED BY (key) ON ("484") STORED AS DIRECTORIES +POSTHOOK: query: ALTER TABLE test_tab_n0 SKEWED BY (key) ON ("484") POSTHOOK: type: ALTERTABLE_SKEWED POSTHOOK: Input: default@test_tab_n0 POSTHOOK: Output: default@test_tab_n0 @@ -45,10 +45,10 @@ Table: test_tab_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5370 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -59,10 +59,7 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/test_tab_n0/part=1/key=484} Storage Desc Params: serialization.format 1 diff --git ql/src/test/results/clientpositive/list_bucket_dml_1.q.out ql/src/test/results/clientpositive/list_bucket_dml_1.q.out index d2308bb06f..aa4896facd 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_1.q.out @@ -1,14 +1,12 @@ PREHOOK: query: create table list_bucketing_dynamic_part_n0 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") -stored as DIRECTORIES PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_dynamic_part_n0 POSTHOOK: query: create table list_bucketing_dynamic_part_n0 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") -stored as DIRECTORIES POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_dynamic_part_n0 @@ -302,7 +300,7 @@ Table: list_bucketing_dynamic_part_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -316,11 +314,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_dynamic_part_n0/ds=2008-04-08/hr=11/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: desc formatted list_bucketing_dynamic_part_n0 partition (ds='2008-04-08', hr='12') @@ -345,7 +340,7 @@ Table: list_bucketing_dynamic_part_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -359,11 +354,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_dynamic_part_n0/ds=2008-04-08/hr=12/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds='2008-04-08' @@ -424,7 +416,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part_n0 - numFiles 2 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_dml_11.q.out ql/src/test/results/clientpositive/list_bucket_dml_11.q.out index 550d378ac0..bb67c15702 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_11.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_11.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part_n3 (key String, value String) partitioned by (ds String, hr String) skewed by (value) on ('val_466','val_287','val_82') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part_n3 POSTHOOK: query: create table list_bucketing_static_part_n3 (key String, value String) partitioned by (ds String, hr String) skewed by (value) on ('val_466','val_287','val_82') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -257,10 +255,10 @@ Table: list_bucketing_static_part_n3 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5522 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -271,11 +269,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [value] Skewed Values: [[val_287], [val_466], [val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[val_287]=/list_bucketing_static_part_n3/ds=2008-04-08/hr=11/value=val_287, [val_466]=/list_bucketing_static_part_n3/ds=2008-04-08/hr=11/value=val_466, [val_82]=/list_bucketing_static_part_n3/ds=2008-04-08/hr=11/value=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -307,7 +302,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part_n3 - numFiles 4 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -315,7 +310,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part_n3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 5522 + totalSize 5293 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_12.q.out ql/src/test/results/clientpositive/list_bucket_dml_12.q.out index 07b07fb094..0411a08473 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_12.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_12.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_mul_col_n0 (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_mul_col_n0 POSTHOOK: query: create table list_bucketing_mul_col_n0 (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -263,10 +261,10 @@ Table: list_bucketing_mul_col_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\",\"col4\":\"true\",\"col5\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 6312 - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### # Storage Information @@ -277,11 +275,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [col2, col4] Skewed Values: [[287, val_287], [466, val_466], [82, val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[287, val_287]=/list_bucketing_mul_col_n0/ds=2008-04-08/hr=11/col2=287/col4=val_287, [466, val_466]=/list_bucketing_mul_col_n0/ds=2008-04-08/hr=11/col2=466/col4=val_466, [82, val_82]=/list_bucketing_mul_col_n0/ds=2008-04-08/hr=11/col2=82/col4=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -315,7 +310,7 @@ STAGE PLANS: columns.types string:string:string:string:string #### A masked pattern was here #### name default.list_bucketing_mul_col_n0 - numFiles 4 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -323,7 +318,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_mul_col_n0 { string col1, string col2, string col3, string col4, string col5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -408,7 +403,7 @@ STAGE PLANS: columns.types string:string:string:string:string #### A masked pattern was here #### name default.list_bucketing_mul_col_n0 - numFiles 4 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -416,7 +411,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_mul_col_n0 { string col1, string col2, string col3, string col4, string col5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_13.q.out ql/src/test/results/clientpositive/list_bucket_dml_13.q.out index 4d1224bad6..cf9cebe24c 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_13.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_13.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_mul_col POSTHOOK: query: create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -263,10 +261,10 @@ Table: list_bucketing_mul_col #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\",\"col4\":\"true\",\"col5\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 6312 - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### # Storage Information @@ -277,11 +275,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [col2, col4] Skewed Values: [[287, val_287], [466, val_466], [82, val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[287, val_287]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=287/col4=val_287, [466, val_466]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=466/col4=val_466, [82, val_82]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=82/col4=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -315,7 +310,7 @@ STAGE PLANS: columns.types string:string:string:string:string #### A masked pattern was here #### name default.list_bucketing_mul_col - numFiles 4 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -323,7 +318,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_14.q.out ql/src/test/results/clientpositive/list_bucket_dml_14.q.out index ed23b0d8b3..a37e58d2aa 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_14.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_14.q.out @@ -1,12 +1,10 @@ PREHOOK: query: create table list_bucketing (key String, value String) skewed by (key) on ("484") -stored as DIRECTORIES PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing POSTHOOK: query: create table list_bucketing (key String, value String) skewed by (key) on ("484") -stored as DIRECTORIES POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing @@ -233,7 +231,7 @@ Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} bucketing_version 2 - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -247,11 +245,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from src @@ -348,7 +343,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 serialization.ddl struct list_bucketing { string key, string value} @@ -370,7 +365,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 serialization.ddl struct list_bucketing { string key, string value} diff --git ql/src/test/results/clientpositive/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/list_bucket_dml_2.q.out index ea621b8e67..c061d73e36 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_2.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part_n4 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part_n4 POSTHOOK: query: create table list_bucketing_static_part_n4 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -312,10 +310,10 @@ Table: list_bucketing_static_part_n4 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -326,11 +324,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part_n4/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part_n4/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -380,7 +375,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part_n4 - numFiles 6 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string @@ -388,7 +383,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part_n4 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_3.q.out ql/src/test/results/clientpositive/list_bucket_dml_3.q.out index 3a6b27be2e..08dbcca626 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_3.q.out @@ -1,8 +1,8 @@ -PREHOOK: query: create table list_bucketing_static_part_n1 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") stored as DIRECTORIES +PREHOOK: query: create table list_bucketing_static_part_n1 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_static_part_n1 -POSTHOOK: query: create table list_bucketing_static_part_n1 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") stored as DIRECTORIES +POSTHOOK: query: create table list_bucketing_static_part_n1 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_static_part_n1 @@ -293,7 +293,7 @@ Table: list_bucketing_static_part_n1 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 1000 rawDataSize 10624 totalSize 11624 @@ -307,11 +307,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_static_part_n1/ds=2008-04-08/hr=11/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds='2008-04-08' @@ -372,7 +369,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part_n1 - numFiles 4 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_dml_4.q.out ql/src/test/results/clientpositive/list_bucket_dml_4.q.out index 823e9e1135..ad65583caa 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_4.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_4.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part_n2 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part_n2 POSTHOOK: query: create table list_bucketing_static_part_n2 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -312,10 +310,10 @@ Table: list_bucketing_static_part_n2 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -326,11 +324,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part_n2/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part_n2/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -751,10 +746,10 @@ Table: list_bucketing_static_part_n2 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10786 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -765,11 +760,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part_n2/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part_n2/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -819,7 +811,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part_n2 - numFiles 4 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string @@ -827,7 +819,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part_n2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10786 + totalSize 10586 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_5.q.out ql/src/test/results/clientpositive/list_bucket_dml_5.q.out index d71834f94b..685773b3c6 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_5.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_5.q.out @@ -1,14 +1,12 @@ PREHOOK: query: create table list_bucketing_dynamic_part_n1 (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_dynamic_part_n1 POSTHOOK: query: create table list_bucketing_dynamic_part_n1 (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_dynamic_part_n1 @@ -302,7 +300,7 @@ Table: list_bucketing_dynamic_part_n1 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -316,11 +314,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part_n1/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part_n1/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: desc formatted list_bucketing_dynamic_part_n1 partition (ds='2008-04-08', hr='12') @@ -345,7 +340,7 @@ Table: list_bucketing_dynamic_part_n1 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -359,11 +354,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part_n1/ds=2008-04-08/hr=12/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part_n1/ds=2008-04-08/hr=12/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds='2008-04-08' @@ -429,7 +421,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part_n1 - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -476,7 +468,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part_n1 - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_dml_6.q.out ql/src/test/results/clientpositive/list_bucket_dml_6.q.out index cd3228e2a8..92fc50ad57 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_6.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_6.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_dynamic_part_n3 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_dynamic_part_n3 POSTHOOK: query: create table list_bucketing_dynamic_part_n3 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -330,7 +328,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] Storage Desc Params: @@ -357,10 +354,10 @@ Table: list_bucketing_dynamic_part_n3 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10734 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -371,11 +368,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part_n3/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part_n3/ds=2008-04-08/hr=b1/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -820,7 +814,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] Storage Desc Params: @@ -847,10 +840,10 @@ Table: list_bucketing_dynamic_part_n3 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10622 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -861,11 +854,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part_n3/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part_n3/ds=2008-04-08/hr=b1/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -962,7 +952,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part_n3 - numFiles 4 + numFiles 2 numRows 984 partition_columns ds/hr partition_columns.types string:string @@ -970,7 +960,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_dynamic_part_n3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10622 + totalSize 10422 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_7.q.out ql/src/test/results/clientpositive/list_bucket_dml_7.q.out index a325337ac4..2187b9bd9f 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_7.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_7.q.out @@ -1,7 +1,6 @@ PREHOOK: query: CREATE TABLE list_bucketing_dynamic_part (key String, value STRING) PARTITIONED BY (ds string, hr string) skewed by (key) on ('484') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_dynamic_part POSTHOOK: query: CREATE TABLE list_bucketing_dynamic_part (key String, value STRING) PARTITIONED BY (ds string, hr string) skewed by (key) on ('484') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -330,7 +328,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] Storage Desc Params: @@ -357,10 +354,10 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10576 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -371,11 +368,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -820,7 +814,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] Storage Desc Params: @@ -847,10 +840,10 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10520 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -861,11 +854,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -962,7 +952,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part - numFiles 3 + numFiles 2 numRows 984 partition_columns ds/hr partition_columns.types string:string @@ -970,7 +960,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_dynamic_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10520 + totalSize 10422 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_8.q.out ql/src/test/results/clientpositive/list_bucket_dml_8.q.out index 3aa3940132..8c27ea0825 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_8.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_8.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_dynamic_part_n2 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_dynamic_part_n2 POSTHOOK: query: create table list_bucketing_dynamic_part_n2 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -330,7 +328,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] Storage Desc Params: @@ -357,10 +354,10 @@ Table: list_bucketing_dynamic_part_n2 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10734 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -371,11 +368,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part_n2/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part_n2/ds=2008-04-08/hr=b1/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: alter table list_bucketing_dynamic_part_n2 partition (ds='2008-04-08', hr='b1') concatenate @@ -407,10 +401,10 @@ Database: default Table: list_bucketing_dynamic_part_n2 #### A masked pattern was here #### Partition Parameters: - numFiles 3 + numFiles 1 numRows 984 rawDataSize 9488 - totalSize 10586 + totalSize 10386 #### A masked pattern was here #### # Storage Information @@ -421,11 +415,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part_n2/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part_n2/ds=2008-04-08/hr=b1/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -525,7 +516,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part_n2 - numFiles 3 + numFiles 1 numRows 984 partition_columns ds/hr partition_columns.types string:string @@ -533,7 +524,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_dynamic_part_n2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10586 + totalSize 10386 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_9.q.out ql/src/test/results/clientpositive/list_bucket_dml_9.q.out index 78f1d8ee8f..73fd02c3e3 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_9.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_9.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part_n0 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','103') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part_n0 POSTHOOK: query: create table list_bucketing_static_part_n0 (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','103') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -312,10 +310,10 @@ Table: list_bucketing_static_part_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -326,11 +324,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[103], [484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part_n0/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part_n0/ds=2008-04-08/hr=11/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -751,10 +746,10 @@ Table: list_bucketing_static_part_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10786 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -765,11 +760,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[103], [484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part_n0/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part_n0/ds=2008-04-08/hr=11/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -819,7 +811,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part_n0 - numFiles 4 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string @@ -827,7 +819,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part_n0 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10786 + totalSize 10586 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out index 40455b771e..4b9715189b 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out @@ -1,14 +1,12 @@ PREHOOK: query: create table fact_daily (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('238','val_238')) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@fact_daily POSTHOOK: query: create table fact_daily (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('238','val_238')) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@fact_daily @@ -46,7 +44,7 @@ Table: fact_daily #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -60,11 +58,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[238, val_238], [484, val_484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[238, val_238]=/fact_daily/ds=1/hr=4/key=238/value=val_238, [484, val_484]=/fact_daily/ds=1/hr=4/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: SELECT count(1) FROM fact_daily WHERE ds='1' and hr='4' @@ -103,7 +98,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -188,7 +183,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -274,7 +269,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -358,7 +353,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out index a7cf4e90c4..70990696c2 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out @@ -1,14 +1,12 @@ PREHOOK: query: create table fact_daily_n2 (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('238','val_238')) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@fact_daily_n2 POSTHOOK: query: create table fact_daily_n2 (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('238','val_238')) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@fact_daily_n2 @@ -46,7 +44,7 @@ Table: fact_daily_n2 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -60,11 +58,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[238, val_238], [484, val_484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[238, val_238]=/fact_daily_n2/ds=1/hr=4/key=238/value=val_238, [484, val_484]=/fact_daily_n2/ds=1/hr=4/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: SELECT count(1) FROM fact_daily_n2 WHERE ds='1' and hr='4' @@ -103,7 +98,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily_n2 - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -188,7 +183,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily_n2 - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -276,7 +271,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily_n2 - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out index 273f2d0b02..e7c669deec 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out @@ -58,11 +58,11 @@ Bucket Columns: [] Sort Columns: [] Storage Desc Params: serialization.format 1 -PREHOOK: query: alter table fact_daily_n3 skewed by (key, value) on (('484','val_484'),('238','val_238')) stored as DIRECTORIES +PREHOOK: query: alter table fact_daily_n3 skewed by (key, value) on (('484','val_484'),('238','val_238')) PREHOOK: type: ALTERTABLE_SKEWED PREHOOK: Input: default@fact_daily_n3 PREHOOK: Output: default@fact_daily_n3 -POSTHOOK: query: alter table fact_daily_n3 skewed by (key, value) on (('484','val_484'),('238','val_238')) stored as DIRECTORIES +POSTHOOK: query: alter table fact_daily_n3 skewed by (key, value) on (('484','val_484'),('238','val_238')) POSTHOOK: type: ALTERTABLE_SKEWED POSTHOOK: Input: default@fact_daily_n3 POSTHOOK: Output: default@fact_daily_n3 @@ -100,7 +100,7 @@ Table: fact_daily_n3 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -114,18 +114,15 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[238, val_238], [484, val_484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[238, val_238]=/fact_daily_n3/ds=1/hr=2/key=238/value=val_238, [484, val_484]=/fact_daily_n3/ds=1/hr=2/key=484/value=val_484} Storage Desc Params: serialization.format 1 -PREHOOK: query: alter table fact_daily_n3 skewed by (key, value) on (('327','val_327')) stored as DIRECTORIES +PREHOOK: query: alter table fact_daily_n3 skewed by (key, value) on (('327','val_327')) PREHOOK: type: ALTERTABLE_SKEWED PREHOOK: Input: default@fact_daily_n3 PREHOOK: Output: default@fact_daily_n3 -POSTHOOK: query: alter table fact_daily_n3 skewed by (key, value) on (('327','val_327')) stored as DIRECTORIES +POSTHOOK: query: alter table fact_daily_n3 skewed by (key, value) on (('327','val_327')) POSTHOOK: type: ALTERTABLE_SKEWED POSTHOOK: Input: default@fact_daily_n3 POSTHOOK: Output: default@fact_daily_n3 @@ -163,7 +160,7 @@ Table: fact_daily_n3 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -177,11 +174,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[327, val_327]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[327, val_327]=/fact_daily_n3/ds=1/hr=3/key=327/value=val_327} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -325,7 +319,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily_n3 - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -412,7 +406,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily_n3 - numFiles 2 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out index 5e9d191885..a4a83181b2 100644 --- ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -271,10 +269,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5520 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -285,10 +283,7 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[103], [484], [51]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484, [51]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=51} Storage Desc Params: serialization.format 1 diff --git ql/src/test/results/clientpositive/llap/mm_all.q.out ql/src/test/results/clientpositive/llap/mm_all.q.out index 95734b6b4f..bfb4675d1f 100644 --- ql/src/test/results/clientpositive/llap/mm_all.q.out +++ ql/src/test/results/clientpositive/llap/mm_all.q.out @@ -778,12 +778,12 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@partunion_mm POSTHOOK: Output: default@partunion_mm PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +tblproperties ("transactional"="true", "transactional_properties"="insert_only") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@skew_mm POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +tblproperties ("transactional"="true", "transactional_properties"="insert_only") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@skew_mm @@ -829,12 +829,12 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@skew_mm POSTHOOK: Output: default@skew_mm PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@skew_dp_union_mm POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@skew_dp_union_mm diff --git ql/src/test/results/clientpositive/llap/resourceplan.q.out ql/src/test/results/clientpositive/llap/resourceplan.q.out index bd55592406..68ab151636 100644 --- ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -851,7 +851,6 @@ PREHOOK: query: CREATE TABLE IF NOT EXISTS `SDS` ( `CD_ID` bigint, `INPUT_FORMAT` string, `IS_COMPRESSED` boolean, - `IS_STOREDASSUBDIRECTORIES` boolean, `LOCATION` string, `NUM_BUCKETS` int, `OUTPUT_FORMAT` string, @@ -867,7 +866,6 @@ TBLPROPERTIES ( \"CD_ID\", \"INPUT_FORMAT\", \"IS_COMPRESSED\", - \"IS_STOREDASSUBDIRECTORIES\", \"LOCATION\", \"NUM_BUCKETS\", \"OUTPUT_FORMAT\", @@ -883,7 +881,6 @@ POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SDS` ( `CD_ID` bigint, `INPUT_FORMAT` string, `IS_COMPRESSED` boolean, - `IS_STOREDASSUBDIRECTORIES` boolean, `LOCATION` string, `NUM_BUCKETS` int, `OUTPUT_FORMAT` string, @@ -899,7 +896,6 @@ TBLPROPERTIES ( \"CD_ID\", \"INPUT_FORMAT\", \"IS_COMPRESSED\", - \"IS_STOREDASSUBDIRECTORIES\", \"LOCATION\", \"NUM_BUCKETS\", \"OUTPUT_FORMAT\", diff --git ql/src/test/results/clientpositive/mm_all.q.out ql/src/test/results/clientpositive/mm_all.q.out index e7df4c0a29..b38de22528 100644 --- ql/src/test/results/clientpositive/mm_all.q.out +++ ql/src/test/results/clientpositive/mm_all.q.out @@ -792,12 +792,12 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@partunion_mm POSTHOOK: Output: default@partunion_mm PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +tblproperties ("transactional"="true", "transactional_properties"="insert_only") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@skew_mm POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +tblproperties ("transactional"="true", "transactional_properties"="insert_only") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@skew_mm @@ -843,12 +843,12 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@skew_mm POSTHOOK: Output: default@skew_mm PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@skew_dp_union_mm POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@skew_dp_union_mm diff --git ql/src/test/results/clientpositive/show_create_table_db_table.q.out ql/src/test/results/clientpositive/show_create_table_db_table.q.out index 33410c8582..4bda1d3098 100644 --- ql/src/test/results/clientpositive/show_create_table_db_table.q.out +++ ql/src/test/results/clientpositive/show_create_table_db_table.q.out @@ -26,11 +26,11 @@ POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt2(key string, value int) skewe POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:tmp_feng POSTHOOK: Output: tmp_feng@tmp_showcrt2 -PREHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') stored as directories +PREHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') PREHOOK: type: CREATETABLE PREHOOK: Output: database:tmp_feng PREHOOK: Output: tmp_feng@tmp_showcrt3 -POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') stored as directories +POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:tmp_feng POSTHOOK: Output: tmp_feng@tmp_showcrt3 @@ -93,7 +93,6 @@ CREATE TABLE `tmp_feng.tmp_showcrt3`( `value` int) SKEWED BY (key) ON (('1'),('2')) - STORED AS DIRECTORIES ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' STORED AS INPUTFORMAT diff --git ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out index e14ae92d1c..d6212564bc 100644 --- ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -207,10 +205,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5520 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -221,10 +219,7 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[103], [484], [51]] -Skewed Value to Path: {[103]=hdfs://### HDFS PATH ###, [484]=hdfs://### HDFS PATH ###, [51]=hdfs://### HDFS PATH ###} -Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484, [51]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=51} Storage Desc Params: serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out index 1dd4884cda..c6ed471e0e 100644 --- ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part_n4 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part_n4 POSTHOOK: query: create table list_bucketing_static_part_n4 (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -258,10 +256,10 @@ Table: list_bucketing_static_part_n4 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 6 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -272,11 +270,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part_n4/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part_n4/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -326,7 +321,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part_n4 - numFiles 6 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string @@ -334,7 +329,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part_n4 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/stats_list_bucket.q.out ql/src/test/results/clientpositive/stats_list_bucket.q.out index 566af31192..946859bb68 100644 --- ql/src/test/results/clientpositive/stats_list_bucket.q.out +++ ql/src/test/results/clientpositive/stats_list_bucket.q.out @@ -11,7 +11,6 @@ PREHOOK: query: create table stats_list_bucket ( c2 string ) partitioned by (ds string, hr string) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -21,7 +20,6 @@ POSTHOOK: query: create table stats_list_bucket ( c2 string ) partitioned by (ds string, hr string) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -60,10 +58,10 @@ Table: stats_list_bucket #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5522 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -74,11 +72,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [c1, c2] Skewed Values: [[287, val_287], [466, val_466], [82, val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[287, val_287]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=287/c2=val_287, [466, val_466]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=466/c2=val_466, [82, val_82]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=82/c2=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: create table stats_list_bucket_1 ( @@ -86,7 +81,6 @@ PREHOOK: query: create table stats_list_bucket_1 ( c2 string ) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -96,7 +90,6 @@ POSTHOOK: query: create table stats_list_bucket_1 ( c2 string ) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -132,10 +125,10 @@ Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\"}} bucketing_version 2 - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5522 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -146,11 +139,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [c1, c2] Skewed Values: [[287, val_287], [466, val_466], [82, val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[287, val_287]=/stats_list_bucket_1/c1=287/c2=val_287, [466, val_466]=/stats_list_bucket_1/c1=466/c2=val_466, [82, val_82]=/stats_list_bucket_1/c1=82/c2=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: drop table stats_list_bucket diff --git ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out index bc355544a3..a8986406f2 100644 --- ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out +++ ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out @@ -8,13 +8,11 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@test_tab_n3 PREHOOK: query: ALTER TABLE test_tab_n3 SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES PREHOOK: type: ALTERTABLE_SKEWED PREHOOK: Input: default@test_tab_n3 PREHOOK: Output: default@test_tab_n3 POSTHOOK: query: ALTER TABLE test_tab_n3 SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES POSTHOOK: type: ALTERTABLE_SKEWED POSTHOOK: Input: default@test_tab_n3 POSTHOOK: Output: default@test_tab_n3 @@ -74,7 +72,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.test_tab_n3 - numFiles 2 + numFiles 1 numRows 500 partition_columns part partition_columns.types string @@ -82,7 +80,7 @@ STAGE PLANS: serialization.ddl struct test_tab_n3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 1761 + totalSize 1685 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -157,7 +155,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.test_tab_n3 - numFiles 2 + numFiles 1 numRows 500 partition_columns part partition_columns.types string @@ -165,7 +163,7 @@ STAGE PLANS: serialization.ddl struct test_tab_n3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 1761 + totalSize 1685 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp index bc4d168a74..3409d1ed54 100644 --- standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp +++ standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp @@ -6008,11 +6008,6 @@ void StorageDescriptor::__set_skewedInfo(const SkewedInfo& val) { __isset.skewedInfo = true; } -void StorageDescriptor::__set_storedAsSubDirectories(const bool val) { - this->storedAsSubDirectories = val; -__isset.storedAsSubDirectories = true; -} - uint32_t StorageDescriptor::read(::apache::thrift::protocol::TProtocol* iprot) { apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); @@ -6173,14 +6168,6 @@ uint32_t StorageDescriptor::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; - case 12: - if (ftype == ::apache::thrift::protocol::T_BOOL) { - xfer += iprot->readBool(this->storedAsSubDirectories); - this->__isset.storedAsSubDirectories = true; - } else { - xfer += iprot->skip(ftype); - } - break; default: xfer += iprot->skip(ftype); break; @@ -6276,11 +6263,6 @@ uint32_t StorageDescriptor::write(::apache::thrift::protocol::TProtocol* oprot) xfer += this->skewedInfo.write(oprot); xfer += oprot->writeFieldEnd(); } - if (this->__isset.storedAsSubDirectories) { - xfer += oprot->writeFieldBegin("storedAsSubDirectories", ::apache::thrift::protocol::T_BOOL, 12); - xfer += oprot->writeBool(this->storedAsSubDirectories); - xfer += oprot->writeFieldEnd(); - } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -6299,7 +6281,6 @@ void swap(StorageDescriptor &a, StorageDescriptor &b) { swap(a.sortCols, b.sortCols); swap(a.parameters, b.parameters); swap(a.skewedInfo, b.skewedInfo); - swap(a.storedAsSubDirectories, b.storedAsSubDirectories); swap(a.__isset, b.__isset); } @@ -6315,7 +6296,6 @@ StorageDescriptor::StorageDescriptor(const StorageDescriptor& other234) { sortCols = other234.sortCols; parameters = other234.parameters; skewedInfo = other234.skewedInfo; - storedAsSubDirectories = other234.storedAsSubDirectories; __isset = other234.__isset; } StorageDescriptor& StorageDescriptor::operator=(const StorageDescriptor& other235) { @@ -6330,7 +6310,6 @@ StorageDescriptor& StorageDescriptor::operator=(const StorageDescriptor& other23 sortCols = other235.sortCols; parameters = other235.parameters; skewedInfo = other235.skewedInfo; - storedAsSubDirectories = other235.storedAsSubDirectories; __isset = other235.__isset; return *this; } @@ -6348,7 +6327,6 @@ void StorageDescriptor::printTo(std::ostream& out) const { out << ", " << "sortCols=" << to_string(sortCols); out << ", " << "parameters=" << to_string(parameters); out << ", " << "skewedInfo="; (__isset.skewedInfo ? (out << to_string(skewedInfo)) : (out << "")); - out << ", " << "storedAsSubDirectories="; (__isset.storedAsSubDirectories ? (out << to_string(storedAsSubDirectories)) : (out << "")); out << ")"; } diff --git standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h index 5c6495e43f..e29b79fb83 100644 --- standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -2985,7 +2985,7 @@ inline std::ostream& operator<<(std::ostream& out, const SkewedInfo& obj) } typedef struct _StorageDescriptor__isset { - _StorageDescriptor__isset() : cols(false), location(false), inputFormat(false), outputFormat(false), compressed(false), numBuckets(false), serdeInfo(false), bucketCols(false), sortCols(false), parameters(false), skewedInfo(false), storedAsSubDirectories(false) {} + _StorageDescriptor__isset() : cols(false), location(false), inputFormat(false), outputFormat(false), compressed(false), numBuckets(false), serdeInfo(false), bucketCols(false), sortCols(false), parameters(false), skewedInfo(false) {} bool cols :1; bool location :1; bool inputFormat :1; @@ -2997,7 +2997,6 @@ typedef struct _StorageDescriptor__isset { bool sortCols :1; bool parameters :1; bool skewedInfo :1; - bool storedAsSubDirectories :1; } _StorageDescriptor__isset; class StorageDescriptor { @@ -3005,7 +3004,7 @@ class StorageDescriptor { StorageDescriptor(const StorageDescriptor&); StorageDescriptor& operator=(const StorageDescriptor&); - StorageDescriptor() : location(), inputFormat(), outputFormat(), compressed(0), numBuckets(0), storedAsSubDirectories(0) { + StorageDescriptor() : location(), inputFormat(), outputFormat(), compressed(0), numBuckets(0) { } virtual ~StorageDescriptor() throw(); @@ -3020,7 +3019,6 @@ class StorageDescriptor { std::vector sortCols; std::map parameters; SkewedInfo skewedInfo; - bool storedAsSubDirectories; _StorageDescriptor__isset __isset; @@ -3046,8 +3044,6 @@ class StorageDescriptor { void __set_skewedInfo(const SkewedInfo& val); - void __set_storedAsSubDirectories(const bool val); - bool operator == (const StorageDescriptor & rhs) const { if (!(cols == rhs.cols)) @@ -3074,10 +3070,6 @@ class StorageDescriptor { return false; else if (__isset.skewedInfo && !(skewedInfo == rhs.skewedInfo)) return false; - if (__isset.storedAsSubDirectories != rhs.__isset.storedAsSubDirectories) - return false; - else if (__isset.storedAsSubDirectories && !(storedAsSubDirectories == rhs.storedAsSubDirectories)) - return false; return true; } bool operator != (const StorageDescriptor &rhs) const { diff --git standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java index 3cfa765fc1..1c7c505b31 100644 --- standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java +++ standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java @@ -49,7 +49,6 @@ private static final org.apache.thrift.protocol.TField SORT_COLS_FIELD_DESC = new org.apache.thrift.protocol.TField("sortCols", org.apache.thrift.protocol.TType.LIST, (short)9); private static final org.apache.thrift.protocol.TField PARAMETERS_FIELD_DESC = new org.apache.thrift.protocol.TField("parameters", org.apache.thrift.protocol.TType.MAP, (short)10); private static final org.apache.thrift.protocol.TField SKEWED_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("skewedInfo", org.apache.thrift.protocol.TType.STRUCT, (short)11); - private static final org.apache.thrift.protocol.TField STORED_AS_SUB_DIRECTORIES_FIELD_DESC = new org.apache.thrift.protocol.TField("storedAsSubDirectories", org.apache.thrift.protocol.TType.BOOL, (short)12); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -68,7 +67,6 @@ private List sortCols; // required private Map parameters; // required private SkewedInfo skewedInfo; // optional - private boolean storedAsSubDirectories; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -82,8 +80,7 @@ BUCKET_COLS((short)8, "bucketCols"), SORT_COLS((short)9, "sortCols"), PARAMETERS((short)10, "parameters"), - SKEWED_INFO((short)11, "skewedInfo"), - STORED_AS_SUB_DIRECTORIES((short)12, "storedAsSubDirectories"); + SKEWED_INFO((short)11, "skewedInfo"); private static final Map byName = new HashMap(); @@ -120,8 +117,6 @@ public static _Fields findByThriftId(int fieldId) { return PARAMETERS; case 11: // SKEWED_INFO return SKEWED_INFO; - case 12: // STORED_AS_SUB_DIRECTORIES - return STORED_AS_SUB_DIRECTORIES; default: return null; } @@ -164,9 +159,8 @@ public String getFieldName() { // isset id assignments private static final int __COMPRESSED_ISSET_ID = 0; private static final int __NUMBUCKETS_ISSET_ID = 1; - private static final int __STOREDASSUBDIRECTORIES_ISSET_ID = 2; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.SKEWED_INFO,_Fields.STORED_AS_SUB_DIRECTORIES}; + private static final _Fields optionals[] = {_Fields.SKEWED_INFO}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -197,8 +191,6 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); tmpMap.put(_Fields.SKEWED_INFO, new org.apache.thrift.meta_data.FieldMetaData("skewedInfo", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SkewedInfo.class))); - tmpMap.put(_Fields.STORED_AS_SUB_DIRECTORIES, new org.apache.thrift.meta_data.FieldMetaData("storedAsSubDirectories", org.apache.thrift.TFieldRequirementType.OPTIONAL, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(StorageDescriptor.class, metaDataMap); } @@ -277,7 +269,6 @@ public StorageDescriptor(StorageDescriptor other) { if (other.isSetSkewedInfo()) { this.skewedInfo = new SkewedInfo(other.skewedInfo); } - this.storedAsSubDirectories = other.storedAsSubDirectories; } public StorageDescriptor deepCopy() { @@ -299,8 +290,6 @@ public void clear() { this.sortCols = null; this.parameters = null; this.skewedInfo = null; - setStoredAsSubDirectoriesIsSet(false); - this.storedAsSubDirectories = false; } public int getColsSize() { @@ -610,28 +599,6 @@ public void setSkewedInfoIsSet(boolean value) { } } - public boolean isStoredAsSubDirectories() { - return this.storedAsSubDirectories; - } - - public void setStoredAsSubDirectories(boolean storedAsSubDirectories) { - this.storedAsSubDirectories = storedAsSubDirectories; - setStoredAsSubDirectoriesIsSet(true); - } - - public void unsetStoredAsSubDirectories() { - __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __STOREDASSUBDIRECTORIES_ISSET_ID); - } - - /** Returns true if field storedAsSubDirectories is set (has been assigned a value) and false otherwise */ - public boolean isSetStoredAsSubDirectories() { - return EncodingUtils.testBit(__isset_bitfield, __STOREDASSUBDIRECTORIES_ISSET_ID); - } - - public void setStoredAsSubDirectoriesIsSet(boolean value) { - __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __STOREDASSUBDIRECTORIES_ISSET_ID, value); - } - public void setFieldValue(_Fields field, Object value) { switch (field) { case COLS: @@ -722,14 +689,6 @@ public void setFieldValue(_Fields field, Object value) { } break; - case STORED_AS_SUB_DIRECTORIES: - if (value == null) { - unsetStoredAsSubDirectories(); - } else { - setStoredAsSubDirectories((Boolean)value); - } - break; - } } @@ -768,9 +727,6 @@ public Object getFieldValue(_Fields field) { case SKEWED_INFO: return getSkewedInfo(); - case STORED_AS_SUB_DIRECTORIES: - return isStoredAsSubDirectories(); - } throw new IllegalStateException(); } @@ -804,8 +760,6 @@ public boolean isSet(_Fields field) { return isSetParameters(); case SKEWED_INFO: return isSetSkewedInfo(); - case STORED_AS_SUB_DIRECTORIES: - return isSetStoredAsSubDirectories(); } throw new IllegalStateException(); } @@ -922,15 +876,6 @@ public boolean equals(StorageDescriptor that) { return false; } - boolean this_present_storedAsSubDirectories = true && this.isSetStoredAsSubDirectories(); - boolean that_present_storedAsSubDirectories = true && that.isSetStoredAsSubDirectories(); - if (this_present_storedAsSubDirectories || that_present_storedAsSubDirectories) { - if (!(this_present_storedAsSubDirectories && that_present_storedAsSubDirectories)) - return false; - if (this.storedAsSubDirectories != that.storedAsSubDirectories) - return false; - } - return true; } @@ -993,11 +938,6 @@ public int hashCode() { if (present_skewedInfo) list.add(skewedInfo); - boolean present_storedAsSubDirectories = true && (isSetStoredAsSubDirectories()); - list.add(present_storedAsSubDirectories); - if (present_storedAsSubDirectories) - list.add(storedAsSubDirectories); - return list.hashCode(); } @@ -1119,16 +1059,6 @@ public int compareTo(StorageDescriptor other) { return lastComparison; } } - lastComparison = Boolean.valueOf(isSetStoredAsSubDirectories()).compareTo(other.isSetStoredAsSubDirectories()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetStoredAsSubDirectories()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.storedAsSubDirectories, other.storedAsSubDirectories); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -1230,12 +1160,6 @@ public String toString() { } first = false; } - if (isSetStoredAsSubDirectories()) { - if (!first) sb.append(", "); - sb.append("storedAsSubDirectories:"); - sb.append(this.storedAsSubDirectories); - first = false; - } sb.append(")"); return sb.toString(); } @@ -1421,14 +1345,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, StorageDescriptor s org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 12: // STORED_AS_SUB_DIRECTORIES - if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { - struct.storedAsSubDirectories = iprot.readBool(); - struct.setStoredAsSubDirectoriesIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1524,11 +1440,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, StorageDescriptor oprot.writeFieldEnd(); } } - if (struct.isSetStoredAsSubDirectories()) { - oprot.writeFieldBegin(STORED_AS_SUB_DIRECTORIES_FIELD_DESC); - oprot.writeBool(struct.storedAsSubDirectories); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1580,10 +1491,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor s if (struct.isSetSkewedInfo()) { optionals.set(10); } - if (struct.isSetStoredAsSubDirectories()) { - optionals.set(11); - } - oprot.writeBitSet(optionals, 12); + oprot.writeBitSet(optionals, 11); if (struct.isSetCols()) { { oprot.writeI32(struct.cols.size()); @@ -1642,15 +1550,12 @@ public void write(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor s if (struct.isSetSkewedInfo()) { struct.skewedInfo.write(oprot); } - if (struct.isSetStoredAsSubDirectories()) { - oprot.writeBool(struct.storedAsSubDirectories); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(12); + BitSet incoming = iprot.readBitSet(11); if (incoming.get(0)) { { org.apache.thrift.protocol.TList _list185 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); @@ -1737,10 +1642,6 @@ public void read(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor st struct.skewedInfo.read(iprot); struct.setSkewedInfoIsSet(true); } - if (incoming.get(11)) { - struct.storedAsSubDirectories = iprot.readBool(); - struct.setStoredAsSubDirectoriesIsSet(true); - } } } diff --git standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php index a29ebb7f59..48f0098fe1 100644 --- standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php +++ standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -6035,10 +6035,6 @@ class StorageDescriptor { * @var \metastore\SkewedInfo */ public $skewedInfo = null; - /** - * @var bool - */ - public $storedAsSubDirectories = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -6111,10 +6107,6 @@ class StorageDescriptor { 'type' => TType::STRUCT, 'class' => '\metastore\SkewedInfo', ), - 12 => array( - 'var' => 'storedAsSubDirectories', - 'type' => TType::BOOL, - ), ); } if (is_array($vals)) { @@ -6151,9 +6143,6 @@ class StorageDescriptor { if (isset($vals['skewedInfo'])) { $this->skewedInfo = $vals['skewedInfo']; } - if (isset($vals['storedAsSubDirectories'])) { - $this->storedAsSubDirectories = $vals['storedAsSubDirectories']; - } } } @@ -6300,13 +6289,6 @@ class StorageDescriptor { $xfer += $input->skip($ftype); } break; - case 12: - if ($ftype == TType::BOOL) { - $xfer += $input->readBool($this->storedAsSubDirectories); - } else { - $xfer += $input->skip($ftype); - } - break; default: $xfer += $input->skip($ftype); break; @@ -6430,11 +6412,6 @@ class StorageDescriptor { $xfer += $this->skewedInfo->write($output); $xfer += $output->writeFieldEnd(); } - if ($this->storedAsSubDirectories !== null) { - $xfer += $output->writeFieldBegin('storedAsSubDirectories', TType::BOOL, 12); - $xfer += $output->writeBool($this->storedAsSubDirectories); - $xfer += $output->writeFieldEnd(); - } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 11affe375b..560bfe892d 100644 --- standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -4297,7 +4297,6 @@ class StorageDescriptor: - sortCols - parameters - skewedInfo - - storedAsSubDirectories """ thrift_spec = ( @@ -4313,10 +4312,9 @@ class StorageDescriptor: (9, TType.LIST, 'sortCols', (TType.STRUCT,(Order, Order.thrift_spec)), None, ), # 9 (10, TType.MAP, 'parameters', (TType.STRING,None,TType.STRING,None), None, ), # 10 (11, TType.STRUCT, 'skewedInfo', (SkewedInfo, SkewedInfo.thrift_spec), None, ), # 11 - (12, TType.BOOL, 'storedAsSubDirectories', None, None, ), # 12 ) - def __init__(self, cols=None, location=None, inputFormat=None, outputFormat=None, compressed=None, numBuckets=None, serdeInfo=None, bucketCols=None, sortCols=None, parameters=None, skewedInfo=None, storedAsSubDirectories=None,): + def __init__(self, cols=None, location=None, inputFormat=None, outputFormat=None, compressed=None, numBuckets=None, serdeInfo=None, bucketCols=None, sortCols=None, parameters=None, skewedInfo=None,): self.cols = cols self.location = location self.inputFormat = inputFormat @@ -4328,7 +4326,6 @@ def __init__(self, cols=None, location=None, inputFormat=None, outputFormat=None self.sortCols = sortCols self.parameters = parameters self.skewedInfo = skewedInfo - self.storedAsSubDirectories = storedAsSubDirectories def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -4419,11 +4416,6 @@ def read(self, iprot): self.skewedInfo.read(iprot) else: iprot.skip(ftype) - elif fid == 12: - if ftype == TType.BOOL: - self.storedAsSubDirectories = iprot.readBool() - else: - iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -4491,10 +4483,6 @@ def write(self, oprot): oprot.writeFieldBegin('skewedInfo', TType.STRUCT, 11) self.skewedInfo.write(oprot) oprot.writeFieldEnd() - if self.storedAsSubDirectories is not None: - oprot.writeFieldBegin('storedAsSubDirectories', TType.BOOL, 12) - oprot.writeBool(self.storedAsSubDirectories) - oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -4515,7 +4503,6 @@ def __hash__(self): value = (value * 31) ^ hash(self.sortCols) value = (value * 31) ^ hash(self.parameters) value = (value * 31) ^ hash(self.skewedInfo) - value = (value * 31) ^ hash(self.storedAsSubDirectories) return value def __repr__(self): diff --git standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb index fc640d07c1..86280ad75a 100644 --- standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -1017,7 +1017,6 @@ class StorageDescriptor SORTCOLS = 9 PARAMETERS = 10 SKEWEDINFO = 11 - STOREDASSUBDIRECTORIES = 12 FIELDS = { COLS => {:type => ::Thrift::Types::LIST, :name => 'cols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}}, @@ -1030,8 +1029,7 @@ class StorageDescriptor BUCKETCOLS => {:type => ::Thrift::Types::LIST, :name => 'bucketCols', :element => {:type => ::Thrift::Types::STRING}}, SORTCOLS => {:type => ::Thrift::Types::LIST, :name => 'sortCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Order}}, PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}, - SKEWEDINFO => {:type => ::Thrift::Types::STRUCT, :name => 'skewedInfo', :class => ::SkewedInfo, :optional => true}, - STOREDASSUBDIRECTORIES => {:type => ::Thrift::Types::BOOL, :name => 'storedAsSubDirectories', :optional => true} + SKEWEDINFO => {:type => ::Thrift::Types::STRUCT, :name => 'skewedInfo', :class => ::SkewedInfo, :optional => true} } def struct_fields; FIELDS; end diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 51e081b22f..3f6ece06de 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -644,7 +644,7 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw "select " + PARTITIONS + ".\"PART_ID\", " + SDS + ".\"SD_ID\", " + SDS + ".\"CD_ID\"," + " " + SERDES + ".\"SERDE_ID\", " + PARTITIONS + ".\"CREATE_TIME\"," + " " + PARTITIONS + ".\"LAST_ACCESS_TIME\", " + SDS + ".\"INPUT_FORMAT\", " + SDS + ".\"IS_COMPRESSED\"," - + " " + SDS + ".\"IS_STOREDASSUBDIRECTORIES\", " + SDS + ".\"LOCATION\", " + SDS + ".\"NUM_BUCKETS\"," + + " " + SDS + ".\"LOCATION\", " + SDS + ".\"NUM_BUCKETS\"," + " " + SDS + ".\"OUTPUT_FORMAT\", " + SERDES + ".\"NAME\", " + SERDES + ".\"SLIB\" " + "from " + PARTITIONS + "" + " left outer join " + SDS + " on " + PARTITIONS + ".\"SD_ID\" = " + SDS + ".\"SD_ID\" " @@ -717,11 +717,9 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw sd.setInputFormat((String)fields[6]); Boolean tmpBoolean = extractSqlBoolean(fields[7]); if (tmpBoolean != null) sd.setCompressed(tmpBoolean); - tmpBoolean = extractSqlBoolean(fields[8]); - if (tmpBoolean != null) sd.setStoredAsSubDirectories(tmpBoolean); - sd.setLocation((String)fields[9]); - if (fields[10] != null) sd.setNumBuckets(extractSqlInt(fields[10])); - sd.setOutputFormat((String)fields[11]); + sd.setLocation((String)fields[8]); + if (fields[10] != null) sd.setNumBuckets(extractSqlInt(fields[9])); + sd.setOutputFormat((String)fields[10]); sdSb.append(sdId).append(","); part.setSd(sd); @@ -743,8 +741,8 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw throw new MetaException("SDs reuse serdes; we don't expect that"); } serde.setParameters(new HashMap()); - serde.setName((String)fields[12]); - serde.setSerializationLib((String)fields[13]); + serde.setName((String)fields[11]); + serde.setSerializationLib((String)fields[12]); serdeSb.append(serdeId).append(","); sd.setSerdeInfo(serde); Deadline.checkTimeout(); diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index e99f888eef..56bc9cd79b 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -2164,7 +2164,6 @@ private StorageDescriptor convertToStorageDescriptor( convertToSkewedValues(msd.getSkewedColValues()), covertToSkewedMap(msd.getSkewedColValueLocationMaps())); sd.setSkewedInfo(skewedInfo); - sd.setStoredAsSubDirectories(msd.isStoredAsSubDirectories()); return sd; } @@ -2275,7 +2274,7 @@ private MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd, convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValues()), covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() - .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories()); + .getSkewedColValueLocationMaps())); } private MCreationMetadata convertToMCreationMetadata( @@ -4311,7 +4310,6 @@ private void copyMSD(MStorageDescriptor newSd, MStorageDescriptor oldSd) { oldSd.setSkewedColValueLocationMaps(newSd.getSkewedColValueLocationMaps()); oldSd.setSortCols(newSd.getSortCols()); oldSd.setParameters(newSd.getParameters()); - oldSd.setStoredAsSubDirectories(newSd.isStoredAsSubDirectories()); } /** diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java index 433e7c7c18..7eb4f0e565 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java @@ -41,7 +41,7 @@ private String location, inputFormat, outputFormat; private int numBuckets; private Map storageDescriptorParams; - private boolean compressed, storedAsSubDirectories; + private boolean compressed; private List bucketCols, skewedColNames; private List sortCols; private List> skewedColValues; @@ -64,7 +64,6 @@ protected StorageDescriptorBuilder() { protected StorageDescriptor buildSd() throws MetaException { StorageDescriptor sd = new StorageDescriptor(getCols(), location, inputFormat, outputFormat, compressed, numBuckets, buildSerde(), bucketCols, sortCols, storageDescriptorParams); - sd.setStoredAsSubDirectories(storedAsSubDirectories); if (skewedColNames != null) { SkewedInfo skewed = new SkewedInfo(skewedColNames, skewedColValues, skewedColValueLocationMaps); @@ -110,11 +109,6 @@ public T setCompressed(boolean compressed) { return child; } - public T setStoredAsSubDirectories(boolean storedAsSubDirectories) { - this.storedAsSubDirectories = storedAsSubDirectories; - return child; - } - public T setBucketCols(List bucketCols) { this.bucketCols = bucketCols; return child; diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MStorageDescriptor.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MStorageDescriptor.java index 4c6ce008f8..91e9dcf856 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MStorageDescriptor.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MStorageDescriptor.java @@ -35,7 +35,6 @@ private List skewedColNames; private List skewedColValues; private Map skewedColValueLocationMaps; - private boolean isStoredAsSubDirectories; public MStorageDescriptor() {} @@ -56,7 +55,7 @@ public MStorageDescriptor(MColumnDescriptor cd, String location, String inputFor String outputFormat, boolean isCompressed, int numBuckets, MSerDeInfo serDeInfo, List bucketCols, List sortOrder, Map parameters, List skewedColNames, List skewedColValues, - Map skewedColValueLocationMaps, boolean storedAsSubDirectories) { + Map skewedColValueLocationMaps) { this.cd = cd; this.location = location; this.inputFormat = inputFormat; @@ -70,7 +69,6 @@ public MStorageDescriptor(MColumnDescriptor cd, String location, String inputFor this.skewedColNames = skewedColNames; this.skewedColValues = skewedColValues; this.skewedColValueLocationMaps = skewedColValueLocationMaps; - this.isStoredAsSubDirectories = storedAsSubDirectories; } @@ -257,21 +255,4 @@ public void setSkewedColValues(List skewedColValues) { public void setSkewedColValueLocationMaps(Map listBucketColValuesMapping) { this.skewedColValueLocationMaps = listBucketColValuesMapping; } - - - /** - * @return the storedAsSubDirectories - */ - public boolean isStoredAsSubDirectories() { - return isStoredAsSubDirectories; - } - - - /** - * @param storedAsSubDirectories the storedAsSubDirectories to set - */ - public void setStoredAsSubDirectories(boolean storedAsSubDirectories) { - this.isStoredAsSubDirectories = storedAsSubDirectories; - } - } diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 16a8c75801..b7025e0624 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -446,7 +446,7 @@ private static String getPartitionValWithInvalidCharacter(List partVals, md.update(e.getValue().getBytes(ENCODING)); } } - md.update(sd.isStoredAsSubDirectories() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING)); + md.update("false".getBytes(ENCODING)); } return md.digest(); diff --git standalone-metastore/src/main/resources/package.jdo standalone-metastore/src/main/resources/package.jdo index 1be3e986a5..4017717049 100644 --- standalone-metastore/src/main/resources/package.jdo +++ standalone-metastore/src/main/resources/package.jdo @@ -381,9 +381,6 @@ - - - diff --git standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql index 352b43e9cd..529d9e42bd 100644 --- standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql +++ standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql @@ -81,7 +81,7 @@ CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VA CREATE TABLE "APP"."PART_COL_PRIVS" ("PART_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_COL_PRIV" VARCHAR(128), "AUTHORIZER" VARCHAR(128)); -CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT, "IS_STOREDASSUBDIRECTORIES" CHAR(1) NOT NULL); +CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT); CREATE TABLE "APP"."SEQUENCE_TABLE" ("SEQUENCE_NAME" VARCHAR(256) NOT NULL, "NEXT_VAL" BIGINT NOT NULL); diff --git standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql index bb691053a5..ab518f8192 100644 --- standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -81,7 +81,7 @@ CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VA CREATE TABLE "APP"."PART_COL_PRIVS" ("PART_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_COL_PRIV" VARCHAR(128), "AUTHORIZER" VARCHAR(128)); -CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT, "IS_STOREDASSUBDIRECTORIES" CHAR(1) NOT NULL); +CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT); CREATE TABLE "APP"."SEQUENCE_TABLE" ("SEQUENCE_NAME" VARCHAR(256) NOT NULL, "NEXT_VAL" BIGINT NOT NULL); diff --git standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql index bc11b4092a..06f0e73fe2 100644 --- standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql +++ standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql @@ -411,7 +411,6 @@ CREATE TABLE SDS CD_ID bigint NULL, INPUT_FORMAT nvarchar(4000) NULL, IS_COMPRESSED bit NOT NULL, - IS_STOREDASSUBDIRECTORIES bit NOT NULL, LOCATION nvarchar(4000) NULL, NUM_BUCKETS int NOT NULL, OUTPUT_FORMAT nvarchar(4000) NULL, diff --git standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 922e8fef38..5527aa2c35 100644 --- standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -412,7 +412,6 @@ CREATE TABLE SDS CD_ID bigint NULL, INPUT_FORMAT nvarchar(4000) NULL, IS_COMPRESSED bit NOT NULL, - IS_STOREDASSUBDIRECTORIES bit NOT NULL, LOCATION nvarchar(4000) NULL, NUM_BUCKETS int NOT NULL, OUTPUT_FORMAT nvarchar(4000) NULL, diff --git standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql index af955dcf7f..f7ef8d4dd0 100644 --- standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql +++ standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql @@ -400,7 +400,6 @@ CREATE TABLE IF NOT EXISTS `SDS` ( `CD_ID` bigint(20) DEFAULT NULL, `INPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `IS_COMPRESSED` bit(1) NOT NULL, - `IS_STOREDASSUBDIRECTORIES` bit(1) NOT NULL, `LOCATION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `NUM_BUCKETS` int(11) NOT NULL, `OUTPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, diff --git standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 6c40e6e4f6..d459dca219 100644 --- standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -400,7 +400,6 @@ CREATE TABLE IF NOT EXISTS `SDS` ( `CD_ID` bigint(20) DEFAULT NULL, `INPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `IS_COMPRESSED` bit(1) NOT NULL, - `IS_STOREDASSUBDIRECTORIES` bit(1) NOT NULL, `LOCATION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `NUM_BUCKETS` int(11) NOT NULL, `OUTPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, diff --git standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql index bc137039c6..dcc797ba80 100644 --- standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql +++ standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql @@ -270,8 +270,7 @@ CREATE TABLE SDS LOCATION VARCHAR2(4000) NULL, NUM_BUCKETS NUMBER (10) NOT NULL, OUTPUT_FORMAT VARCHAR2(4000) NULL, - SERDE_ID NUMBER NULL, - IS_STOREDASSUBDIRECTORIES NUMBER(1) NOT NULL CHECK (IS_STOREDASSUBDIRECTORIES IN (1,0)) + SERDE_ID NUMBER NULL ); ALTER TABLE SDS ADD CONSTRAINT SDS_PK PRIMARY KEY (SD_ID); diff --git standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index e12150a438..47075142db 100644 --- standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -270,8 +270,7 @@ CREATE TABLE SDS LOCATION VARCHAR2(4000) NULL, NUM_BUCKETS NUMBER (10) NOT NULL, OUTPUT_FORMAT VARCHAR2(4000) NULL, - SERDE_ID NUMBER NULL, - IS_STOREDASSUBDIRECTORIES NUMBER(1) NOT NULL CHECK (IS_STOREDASSUBDIRECTORIES IN (1,0)) + SERDE_ID NUMBER NULL ); ALTER TABLE SDS ADD CONSTRAINT SDS_PK PRIMARY KEY (SD_ID); diff --git standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql index 36bab70eda..ad1d4328f3 100644 --- standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql +++ standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql @@ -299,8 +299,7 @@ CREATE TABLE "SDS" ( "NUM_BUCKETS" bigint NOT NULL, "OUTPUT_FORMAT" character varying(4000) DEFAULT NULL::character varying, "SERDE_ID" bigint, - "CD_ID" bigint, - "IS_STOREDASSUBDIRECTORIES" boolean NOT NULL + "CD_ID" bigint ); diff --git standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index b73e1d19f6..f8a5e1b8bf 100644 --- standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -299,8 +299,7 @@ CREATE TABLE "SDS" ( "NUM_BUCKETS" bigint NOT NULL, "OUTPUT_FORMAT" character varying(4000) DEFAULT NULL::character varying, "SERDE_ID" bigint, - "CD_ID" bigint, - "IS_STOREDASSUBDIRECTORIES" boolean NOT NULL + "CD_ID" bigint ); diff --git standalone-metastore/src/main/thrift/hive_metastore.thrift standalone-metastore/src/main/thrift/hive_metastore.thrift index 6e503eb908..b136a604fe 100644 --- standalone-metastore/src/main/thrift/hive_metastore.thrift +++ standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -407,8 +407,7 @@ struct StorageDescriptor { 8: list bucketCols, // reducer grouping columns and clustering columns and bucketing columns` 9: list sortCols, // sort order of the data in each bucket 10: map parameters, // any user supplied key value hash - 11: optional SkewedInfo skewedInfo, // skewed information - 12: optional bool storedAsSubDirectories // stored as subdirectories or not + 11: optional SkewedInfo skewedInfo } // table information diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index cb32236d54..d8c2e8cd70 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -1857,7 +1857,6 @@ public void testComplexTable() throws Exception { assertEquals(tbl2.getTableName(), tblName); assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size()); assertFalse(tbl2.getSd().isCompressed()); - assertFalse(tbl2.getSd().isStoredAsSubDirectories()); assertEquals(tbl2.getSd().getNumBuckets(), 1); assertEquals("Use this for comments etc", tbl2.getSd().getParameters() diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index 62ed380dfc..ee088f5f68 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -52,14 +52,11 @@ import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; -import jline.internal.Log; - import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; @Category(MetastoreCheckinTest.class) @@ -1054,7 +1051,6 @@ private Table createTestTbl(String dbName, String tblName, String tblOwner, SerDeInfo serdeInfo = new SerDeInfo("serde", "seriallib", new HashMap<>()); StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serdeInfo, null, null, serdeParams); - sd.setStoredAsSubDirectories(false); Table tbl = new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, ptnCols, tblParams, null, null, TableType.MANAGED_TABLE.toString()); tbl.setCatName(DEFAULT_CATALOG_NAME); diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java index bf559b46c6..6de2d54902 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java @@ -1442,7 +1442,6 @@ private Table createTable(String dbName, String tableName, List par .setPartCols(partCols) .addStorageDescriptorParam("partTestSDParamKey", "partTestSDParamValue") .setSerdeName(tableName) - .setStoredAsSubDirectories(false) .addSerdeParam("partTestSerdeParamKey", "partTestSerdeParamValue") .setLocation(location) .create(client, metaStore.getConf()); @@ -1582,8 +1581,6 @@ private void verifyPartitionAttributesDefaultValues(Partition partition, String "org.apache.hadoop.hive.ql.io.HiveOutputFormat", sd.getOutputFormat()); Assert.assertFalse("The compressed attribute doesn't have the default value.", sd.isCompressed()); - Assert.assertFalse("The storedAsSubDirectories attribute doesn't have the default value.", - sd.isStoredAsSubDirectories()); Assert.assertEquals("The numBuckets attribute doesn't have the default value.", 0, sd.getNumBuckets()); Assert.assertTrue("The default value of the attribute 'bucketCols' should be an empty list.", diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java index 4f11a55e35..9df9422eb1 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java @@ -920,7 +920,6 @@ private Table createTable(String dbName, String tableName, List par .setPartCols(partCols) .addStorageDescriptorParam("partTestSDParamKey", "partTestSDParamValue") .setSerdeName(tableName) - .setStoredAsSubDirectories(false) .addSerdeParam("partTestSerdeParamKey", "partTestSerdeParamValue") .setLocation(location) .create(client, metaStore.getConf()); diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java index e1c3dcb47f..41276d5791 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java @@ -302,7 +302,6 @@ public void testCreateTableDefaultValues() throws Exception { Assert.assertEquals("Storage descriptor bucket cols", 0, createdSd.getBucketCols().size()); Assert.assertEquals("Storage descriptor sort cols", 0, createdSd.getSortCols().size()); Assert.assertEquals("Storage descriptor parameters", 0, createdSd.getParameters().size()); - Assert.assertFalse("Storage descriptor stored as subdir", createdSd.isStoredAsSubDirectories()); // Serde info SerDeInfo serDeInfo = createdSd.getSerdeInfo(); @@ -1363,7 +1362,6 @@ private Table getTableWithAllParametersSet() throws MetaException { .setViewOriginalText("viewOriginalText") .setSerdeLib("serdelib") .setSerdeName("serdename") - .setStoredAsSubDirectories(true) .addSerdeParam("serdeParam", "serdeParamValue") .addTableParam("tableParam", "tableParamValue") .addStorageDescriptorParam("sdParam", "sdParamValue") diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java index 9e425cff06..75722878e6 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java @@ -341,10 +341,10 @@ public void testValidateLocations() throws Exception { "insert into CTLGS values (1, 'mycat', 'mydescription', 'hdfs://myhost.com:8020/user/hive/warehouse');", "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');", "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'mycat');", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n');", @@ -371,17 +371,17 @@ public void testValidateLocations() throws Exception { "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'mycat');", "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'mycat');", "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'mycat');", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3000 ,1435255431,2,0 ,'hive',0,3000,'mytal3000','MANAGED_TABLE',NULL,NULL,'n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4011 ,1435255431,4,0 ,'hive',0,4001,'mytal4011','MANAGED_TABLE',NULL,NULL,'n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,4,0 ,'hive',0,4002,'','MANAGED_TABLE',NULL,NULL,'n');", @@ -469,8 +469,8 @@ private void createTestHiveTableSchemas() throws IOException { String[] scripts = new String[] { "insert into CTLGS values (1, 'mycat', 'my description', 'hdfs://myhost.com:8020/user/hive/warehouse');", "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,2,'aTable','MANAGED_TABLE',NULL,NULL,'n');", "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);"