diff --git itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q index d4f0c7166f..e1c8636cde 100644 --- itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q +++ itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q @@ -36,7 +36,7 @@ PARTITIONED BY ( pt string, dt string, hr string) -SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +SKEWED BY (id) ON ('1', '2', '3') LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_list_bucketing'; INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id, name, dept FROM empty; diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out index ccd9ba5400..ce54a6d0d1 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out @@ -80,7 +80,7 @@ PARTITIONED BY ( pt string, dt string, hr string) -SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +SKEWED BY (id) ON ('1', '2', '3') #### A masked pattern was here #### PREHOOK: type: CREATETABLE PREHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing @@ -94,7 +94,7 @@ PARTITIONED BY ( pt string, dt string, hr string) -SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +SKEWED BY (id) ON ('1', '2', '3') #### A masked pattern was here #### POSTHOOK: type: CREATETABLE POSTHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java index 9b50fd4f30..5c21d84365 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java @@ -107,7 +107,7 @@ protected void setUp() { db.createTable(src, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); db.loadTable(hadoopDataFile[i], src, - LoadFileType.KEEP_EXISTING, false, false, false, false, null, 0, false); + LoadFileType.KEEP_EXISTING, false, false, false, null, 0, false); i++; } diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java index 3b22f152e8..cc86eb3271 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java @@ -644,10 +644,10 @@ public void testValidateLocations() throws Exception { "insert into CTLGS values(3, 'test_cat_2', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')", "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2')", "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'test_cat_2')", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n')", @@ -674,17 +674,17 @@ public void testValidateLocations() throws Exception { "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'test_cat_2')", "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'test_cat_2')", "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'test_cat_2')", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')", "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3000 ,1435255431,2,0 ,'hive',0,3000,'mytal3000','MANAGED_TABLE',NULL,NULL,'n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4011 ,1435255431,4,0 ,'hive',0,4001,'mytal4011','MANAGED_TABLE',NULL,NULL,'n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,4,0 ,'hive',0,4002,'','MANAGED_TABLE',NULL,NULL,'n')", @@ -773,8 +773,8 @@ private void createTestHiveTableSchemas() throws IOException { String[] scripts = new String[] { "insert into CTLGS values(2, 'my_catalog', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')", "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8021/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'my_catalog')", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,2,'aTable','MANAGED_TABLE',NULL,NULL,'n')", "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)" diff --git metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql index d9606d8495..d1be5edad4 100644 --- metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql +++ metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql @@ -354,7 +354,6 @@ CREATE TABLE IF NOT EXISTS `SDS` ( `CD_ID` bigint, `INPUT_FORMAT` string, `IS_COMPRESSED` boolean, - `IS_STOREDASSUBDIRECTORIES` boolean, `LOCATION` string, `NUM_BUCKETS` int, `OUTPUT_FORMAT` string, @@ -370,7 +369,6 @@ TBLPROPERTIES ( \"CD_ID\", \"INPUT_FORMAT\", \"IS_COMPRESSED\", - \"IS_STOREDASSUBDIRECTORIES\", \"LOCATION\", \"NUM_BUCKETS\", \"OUTPUT_FORMAT\", diff --git ql/pom.xml ql/pom.xml index fedb5f1f80..d5238094c5 100644 --- ql/pom.xml +++ ql/pom.xml @@ -819,7 +819,7 @@ ${basedir}/src/java ${basedir}/src/test - + org.antlr antlr3-maven-plugin diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 8baf309e7f..4ea567f6b4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -246,8 +246,6 @@ INVALID_TABLE_IN_ON_CLAUSE_OF_MERGE(10149, "No columns from target table ''{0}'' found in ON " + "clause ''{1}'' of MERGE statement.", true), - LOAD_INTO_STORED_AS_DIR(10195, "A stored-as-directories table cannot be used as target for LOAD"), - ALTER_TBL_STOREDASDIR_NOT_SKEWED(10196, "This operation is only valid on skewed table."), ALTER_TBL_SKEWED_LOC_NO_LOC(10197, "Alter table skewed location doesn't have locations."), ALTER_TBL_SKEWED_LOC_NO_MAP(10198, "Alter table skewed location doesn't have location map."), SKEWED_TABLE_NO_COLUMN_NAME(10200, "No skewed column name."), diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 63fe8adc8b..1d621222f5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -987,7 +987,6 @@ private DataOutputStream getOutputStream(Path outputFile) throws HiveException { private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc, DriverContext driverContext) throws HiveException { ListBucketingCtx lbCtx = mergeFilesDesc.getLbCtx(); - boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel(); // merge work only needs input and output. @@ -1000,7 +999,6 @@ private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc, pathToAliases.put(mergeFilesDesc.getInputDir().get(0), inputDirstr); mergeWork.setPathToAliases(pathToAliases); mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx()); - mergeWork.resolveConcatenateMerge(db.getConf()); mergeWork.setMapperCannotSpanPartns(true); mergeWork.setSourceTableInputFormat(mergeFilesDesc.getInputFormatClass().getName()); final FileMergeDesc fmd; @@ -1013,7 +1011,7 @@ private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc, fmd.setDpCtx(null); fmd.setHasDynamicPartitions(false); - fmd.setListBucketingAlterTableConcatenate(lbatc); + fmd.setListBucketingAlterTableConcatenate(false); fmd.setListBucketingDepth(lbd); fmd.setOutputPath(mergeFilesDesc.getOutputDir()); @@ -2673,9 +2671,6 @@ else if (sortCol.getOrder() == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_DESC) { colValueList.add("('" + StringUtils.join(colValues, "','") + "')"); } tbl_skewedinfo.append(StringUtils.join(colValueList, ",") + ")"); - if (tbl.isStoredAsSubDirectories()) { - tbl_skewedinfo.append("\n STORED AS DIRECTORIES"); - } } // Row format (SerDe) @@ -4328,8 +4323,6 @@ private static StorageDescriptor retrieveStorageDescriptor(Table tbl, Partition tbl.setSkewedColNames(skewedColNames); tbl.setSkewedColValues(skewedValues); } - - tbl.setStoredAsSubDirectories(alterTbl.isStoredAsSubDirectories()); } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.OWNER) { if (alterTbl.getOwnerPrincipal() != null) { tbl.setOwner(alterTbl.getOwnerPrincipal().getName()); @@ -4405,24 +4398,6 @@ private static StorageDescriptor retrieveStorageDescriptor(Table tbl, Partition return result; } - private void checkMmLb(Table tbl) throws HiveException { - if (!tbl.isStoredAsSubDirectories()) { - return; - } - // TODO [MM gap?]: by design; no-one seems to use LB tables. They will work, but not convert. - // It's possible to work around this by re-creating and re-inserting the table. - throw new HiveException("Converting list bucketed tables stored as subdirectories " - + " to MM is not supported. Please re-create a table in the desired format."); - } - - private void checkMmLb(Partition part) throws HiveException { - if (!part.isStoredAsSubDirectories()) { - return; - } - throw new HiveException("Converting list bucketed tables stored as subdirectories " - + " to MM is not supported. Please re-create a table in the desired format."); - } - private List> generateAddMmTasks(Table tbl, Long writeId) throws HiveException { // We will move all the files in the table/partition directories into the first MM // directory, then commit the first write ID. @@ -4441,7 +4416,6 @@ private void checkMmLb(Partition part) throws HiveException { Iterator partIter = parts.iterator(); while (partIter.hasNext()) { Partition part = partIter.next(); - checkMmLb(part); Path src = part.getDataLocation(), tgt = new Path(src, mmDir); srcs.add(src); tgts.add(tgt); @@ -4450,7 +4424,6 @@ private void checkMmLb(Partition part) throws HiveException { } } } else { - checkMmLb(tbl); Path src = tbl.getDataLocation(), tgt = new Path(src, mmDir); srcs.add(src); tgts.add(tgt); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 01a5b4c9c3..4d155cb7a0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.io.Serializable; -import java.io.StringWriter; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -31,8 +30,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; - -import com.google.common.collect.Lists; +import java.util.function.BiFunction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -41,8 +39,8 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -68,7 +66,10 @@ import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.stats.StatsCollectionContext; import org.apache.hadoop.hive.ql.stats.StatsPublisher; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.Serializer; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; @@ -82,26 +83,12 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hive.common.util.HiveStringUtils; -import org.apache.hive.common.util.Murmur3; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.function.BiFunction; - -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_TEMPORARY_TABLE_STORAGE; +import com.google.common.collect.Lists; /** * File Sink operator implementation. @@ -129,7 +116,6 @@ protected transient RecordWriter[] rowOutWriters; // row specific RecordWriters protected transient int maxPartitions; protected transient ListBucketingCtx lbCtx; - protected transient boolean isSkewedStoredAsSubDirectories; protected transient boolean[] statsFromRecordWriter; protected transient boolean isCollectRWStats; private transient FSPaths prevFsp; @@ -240,7 +226,7 @@ private void commit(FileSystem fs, List commitPaths) throws HiveException private void commitOneOutPath(int idx, FileSystem fs, List commitPaths) throws IOException, HiveException { - if ((bDynParts || isSkewedStoredAsSubDirectories) + if ((bDynParts) && !fs.exists(finalPaths[idx].getParent())) { if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("commit making path for dyn/skew: " + finalPaths[idx].getParent()); @@ -291,13 +277,12 @@ public void abortWriters(FileSystem fs, boolean abort, boolean delete) throws Hi } } - public void initializeBucketPaths(int filesIdx, String taskId, boolean isNativeTable, - boolean isSkewedStoredAsSubDirectories) { + public void initializeBucketPaths(int filesIdx, String taskId, boolean isNativeTable) { if (isNativeTable) { String extension = Utilities.getFileExtension(jc, isCompressed, hiveOutputFormat); String taskWithExt = extension == null ? taskId : taskId + extension; if (!isMmTable) { - if (!bDynParts && !isSkewedStoredAsSubDirectories) { + if (!bDynParts) { finalPaths[filesIdx] = new Path(parent, taskWithExt); } else { finalPaths[filesIdx] = new Path(buildTmpPath(), taskWithExt); @@ -526,13 +511,9 @@ protected void initializeOp(Configuration hconf) throws HiveException { dpSetup(); } - if (lbCtx != null) { - lbSetup(); - } - if (!bDynParts) { fsp = new FSPaths(specPath, conf.isMmTable()); - fsp.subdirAfterTxn = combinePathFragments(generateListBucketingDirName(null), unionPath); + fsp.subdirAfterTxn = unionPath; if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("creating new paths " + System.identityHashCode(fsp) + " from ctor; childSpec " + unionPath + ": tmpPath " + fsp.buildTmpPath() @@ -542,9 +523,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { // Create all the files - this is required because empty files need to be created for // empty buckets // createBucketFiles(fsp); - if (!this.isSkewedStoredAsSubDirectories) { - valToPaths.put("", fsp); // special entry for non-DP case - } + valToPaths.put("", fsp); } final StoragePolicyValue tmpStorage = StoragePolicyValue.lookup(HiveConf @@ -616,13 +595,6 @@ private void logOutputFormatError(Configuration hconf, HiveException ex) { LOG.error(errorWriter.toString(), ex); } - /** - * Initialize list bucketing information - */ - private void lbSetup() { - this.isSkewedStoredAsSubDirectories = ((lbCtx == null) ? false : lbCtx.isSkewedStoredAsDir()); - } - /** * Set up for dynamic partitioning including a new ObjectInspector for the output row. */ @@ -714,7 +686,7 @@ protected void createBucketFiles(FSPaths fsp) throws HiveException { protected void createBucketForFileIdx(FSPaths fsp, int filesIdx) throws HiveException { try { - fsp.initializeBucketPaths(filesIdx, taskId, isNativeTable(), isSkewedStoredAsSubDirectories); + fsp.initializeBucketPaths(filesIdx, taskId, isNativeTable()); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("createBucketForFileIdx " + filesIdx + ": final path " + fsp.finalPaths[filesIdx] + "; out path " + fsp.outPaths[filesIdx] +" (spec path " + specPath + ", tmp path " @@ -850,18 +822,9 @@ protected boolean updateProgress() { @Override public void process(Object row, int tag) throws HiveException { runTimeNumRows++; - /* Create list bucketing sub-directory only if stored-as-directories is on. */ - String lbDirName = null; - lbDirName = (lbCtx == null) ? null : generateListBucketingDirName(row); if (!bDynParts && !filesCreated) { - if (lbDirName != null) { - if (valToPaths.get(lbDirName) == null) { - createNewPaths(null, lbDirName); - } - } else { - createBucketFiles(fsp); - } + createBucketFiles(fsp); } try { @@ -901,19 +864,13 @@ public void process(Object row, int tag) throws HiveException { dpCtx.getWhiteListPattern().toString() + "'. " + "(configure with " + HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.varname + ")"); } - fpaths = getDynOutPaths(dpVals, lbDirName); + fpaths = getDynOutPaths(dpVals); // use SubStructObjectInspector to serialize the non-partitioning columns in the input row recordValue = serializer.serialize(row, subSetOI); } else { - if (lbDirName != null) { - fpaths = valToPaths.get(lbDirName); - if (fpaths == null) { - fpaths = createNewPaths(null, lbDirName); - } - } else { - fpaths = fsp; - } + fpaths = fsp; + recordValue = serializer.serialize(row, inputObjInspectors[0]); // if serializer is ThriftJDBCBinarySerDe, then recordValue is null if the buffer is not full (the size of buffer // is kept track of in the SerDe) @@ -1064,11 +1021,11 @@ assert getConf().getWriteType() != AcidUtils.Operation.DELETE && * @return * @throws HiveException */ - private FSPaths createNewPaths(String dpDir, String lbDir) throws HiveException { + private FSPaths createNewPaths(String dpDir) throws HiveException { FSPaths fsp2 = new FSPaths(specPath, conf.isMmTable()); - fsp2.subdirAfterTxn = combinePathFragments(lbDir, unionPath); + fsp2.subdirAfterTxn = unionPath; fsp2.subdirBeforeTxn = dpDir; - String pathKey = combinePathFragments(dpDir, lbDir); + String pathKey = dpDir; if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("creating new paths {} for {}, childSpec {}: tmpPath {}," + " task path {}", System.identityHashCode(fsp2), pathKey, unionPath, @@ -1085,67 +1042,7 @@ private FSPaths createNewPaths(String dpDir, String lbDir) throws HiveException return fsp2; } - private static String combinePathFragments(String first, String second) { - return first == null ? second : (second == null ? first : first + Path.SEPARATOR + second); - } - - /** - * Generate list bucketing directory name from a row. - * @param row row to process. - * @return directory name. - */ - protected String generateListBucketingDirName(Object row) { - if (!this.isSkewedStoredAsSubDirectories) { - return null; - } - - String lbDirName = null; - List skewedCols = lbCtx.getSkewedColNames(); - List> allSkewedVals = lbCtx.getSkewedColValues(); - Map, String> locationMap = lbCtx.getLbLocationMap(); - - if (row != null) { - List standObjs = new ArrayList(); - List skewedValsCandidate = null; - /* Convert input row to standard objects. */ - ObjectInspectorUtils.copyToStandardObject(standObjs, row, - (StructObjectInspector) inputObjInspectors[0], ObjectInspectorCopyOption.WRITABLE); - - assert (standObjs.size() >= skewedCols.size()) : - "The row has less number of columns than no. of skewed column."; - - skewedValsCandidate = new ArrayList(skewedCols.size()); - for (SkewedColumnPositionPair posPair : lbCtx.getRowSkewedIndex()) { - skewedValsCandidate.add(posPair.getSkewColPosition(), - standObjs.get(posPair.getTblColPosition()).toString()); - } - /* The row matches skewed column names. */ - if (allSkewedVals.contains(skewedValsCandidate)) { - /* matches skewed values. */ - lbDirName = FileUtils.makeListBucketingDirName(skewedCols, skewedValsCandidate); - locationMap.put(skewedValsCandidate, lbDirName); - } else { - lbDirName = createDefaultLbDir(skewedCols, locationMap); - } - } else { - lbDirName = createDefaultLbDir(skewedCols, locationMap); - } - return lbDirName; - } - - private String createDefaultLbDir(List skewedCols, - Map, String> locationMap) { - String lbDirName; - lbDirName = FileUtils.makeDefaultListBucketingDirName(skewedCols, - lbCtx.getDefaultDirName()); - List defaultKey = Lists.newArrayList(lbCtx.getDefaultKey()); - if (!locationMap.containsKey(defaultKey)) { - locationMap.put(defaultKey, lbDirName); - } - return lbDirName; - } - - protected FSPaths getDynOutPaths(List row, String lbDir) throws HiveException { + protected FSPaths getDynOutPaths(List row) throws HiveException { FSPaths fp; @@ -1154,12 +1051,11 @@ protected FSPaths getDynOutPaths(List row, String lbDir) throws HiveExce String pathKey = null; if (dpDir != null) { - String dpAndLbDir = combinePathFragments(dpDir, lbDir); - pathKey = dpAndLbDir; + pathKey = dpDir; if (conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED)) { String buckNum = row.get(row.size() - 1); taskId = Utilities.replaceTaskIdFromFilename(taskId, buckNum); - pathKey = dpAndLbDir + Path.SEPARATOR + taskId; + pathKey = dpDir + Path.SEPARATOR + taskId; } FSPaths fsp2 = valToPaths.get(pathKey); @@ -1203,7 +1099,7 @@ protected FSPaths getDynOutPaths(List row, String lbDir) throws HiveExce prevFsp = null; } - fsp2 = createNewPaths(dpDir, lbDir); + fsp2 = createNewPaths(dpDir); if (prevFsp == null) { prevFsp = fsp2; } @@ -1505,50 +1401,10 @@ private void publishStats() throws HiveException { * This is server side code to create key in order to save statistics to stats database. * Client side will read it via StatsTask.java aggregateStats(). * Client side reads it via db query prefix which is based on partition spec. - * Since store-as-subdir information is not part of partition spec, we have to - * remove store-as-subdir information from variable "keyPrefix" calculation. - * But we have to keep store-as-subdir information in variable "key" calculation - * since each skewed value has a row in stats db and "key" is db key, - * otherwise later value overwrites previous value. - * Performance impact due to string handling is minimum since this method is - * only called once in FileSinkOperator closeOp(). - * For example, - * create table test skewed by (key, value) on (('484','val_484') stored as DIRECTORIES; - * skewedValueDirList contains 2 elements: - * 1. key=484/value=val_484 - * 2. HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * Case #1: Static partition with store-as-sub-dir - * spSpec has SP path - * fspKey has either - * key=484/value=val_484 or - * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * After filter, fspKey is empty, storedAsDirPostFix has either - * key=484/value=val_484 or - * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * so, at the end, "keyPrefix" doesnt have subdir information but "key" has - * Case #2: Dynamic partition with store-as-sub-dir. Assume dp part is hr - * spSpec has SP path - * fspKey has either - * hr=11/key=484/value=val_484 or - * hr=11/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * After filter, fspKey is hr=11, storedAsDirPostFix has either - * key=484/value=val_484 or - * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - * so, at the end, "keyPrefix" doesn't have subdir information from skewed but "key" has * - * In a word, fspKey is consists of DP(dynamic partition spec) + LB(list bucketing spec) - * In stats publishing, full partition spec consists of prefix part of stat key - * but list bucketing spec is regarded as a postfix of stat key. So we split it here. + * In a word, fspKey is consists of DP(dynamic partition spec). */ private String[] splitKey(String fspKey) { - if (!fspKey.isEmpty() && isSkewedStoredAsSubDirectories) { - for (String dir : lbCtx.getSkewedValuesDirNames()) { - int index = fspKey.indexOf(dir); - if (index >= 0) { - return new String[] {fspKey.substring(0, index), fspKey.substring(index + 1)}; - } - } - } return new String[] {fspKey, null}; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index dbda5fdef4..207973c591 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -371,7 +371,7 @@ public int execute(DriverContext driverContext) { + " into " + tbd.getTable().getTableName()); } db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getLoadFileType(), - work.isSrcLocal(), isSkewedStoredAsDirs(tbd), isFullAcidOp, hasFollowingStatsTask(), + work.isSrcLocal(), isFullAcidOp, hasFollowingStatsTask(), tbd.getWriteId(), tbd.getStmtId(), tbd.isInsertOverwrite()); if (work.getOutputs() != null) { DDLTask.addIfAbsentByName(new WriteEntity(table, @@ -470,11 +470,9 @@ private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, db.loadPartition(tbd.getSourcePath(), db.getTable(tbd.getTable().getTableName()), tbd.getPartitionSpec(), tbd.getLoadFileType(), tbd.getInheritTableSpecs(), - isSkewedStoredAsDirs(tbd), work.isSrcLocal(), - work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && - !tbd.isMmTable(), - hasFollowingStatsTask(), - tbd.getWriteId(), tbd.getStmtId(), tbd.isInsertOverwrite()); + work.isSrcLocal(), + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && !tbd.isMmTable(), + hasFollowingStatsTask(), tbd.getWriteId(), tbd.getStmtId(), tbd.isInsertOverwrite()); Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); // See the comment inside updatePartitionBucketSortColumns. @@ -705,11 +703,6 @@ private void checkFileFormats(Hive db, LoadTableDesc tbd, Table table) } } - private boolean isSkewedStoredAsDirs(LoadTableDesc tbd) { - return (tbd.getLbCtx() == null) ? false : tbd.getLbCtx() - .isSkewedStoredAsDir(); - } - /** * Alters the bucketing and/or sorting columns of the partition provided they meet some * validation criteria, e.g. the number of buckets match the number of files, and the diff --git ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java index 07abd378c5..0f44142848 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileWork.java @@ -20,10 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; @@ -38,9 +35,7 @@ import org.apache.hadoop.hive.ql.plan.Explain.Level; import org.apache.hadoop.mapred.InputFormat; -import java.io.IOException; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; @Explain(displayName = "Merge File Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) @@ -142,57 +137,6 @@ public void resolveDynamicPartitionStoredAsSubDirsMerge(HiveConf conf, inputPaths.add(path); } - /** - * alter table ... concatenate - *

- * If it is skewed table, use subdirectories in inputpaths. - */ - public void resolveConcatenateMerge(HiveConf conf) { - isListBucketingAlterTableConcatenate = - ((listBucketingCtx == null) ? false : listBucketingCtx - .isSkewedStoredAsDir()); - LOG.info("isListBucketingAlterTableConcatenate : " + - isListBucketingAlterTableConcatenate); - if (isListBucketingAlterTableConcatenate) { - // use sub-dir as inputpath. - assert ((this.inputPaths != null) && (this.inputPaths.size() == 1)) : - "alter table ... concatenate should only have one" + - " directory inside inputpaths"; - Path dirPath = inputPaths.get(0); - try { - FileSystem inpFs = dirPath.getFileSystem(conf); - List status = HiveStatsUtils.getFileStatusRecurse( - dirPath, listBucketingCtx.getSkewedColNames().size(), inpFs); - List newInputPath = new ArrayList(); - boolean succeed = true; - for (FileStatus s : status) { - if (s.isDir()) { - // Add the lb path to the list of input paths - newInputPath.add(s.getPath()); - } else { - // find file instead of dir. dont change inputpath - succeed = false; - } - } - assert (succeed || ((!succeed) && newInputPath.isEmpty())) : - "This partition has " - + " inconsistent file structure: " - + - "it is stored-as-subdir and expected all files in the same depth" - + " of subdirectories."; - if (succeed) { - inputPaths.clear(); - inputPaths.addAll(newInputPath); - } - } catch (IOException e) { - String msg = - "Fail to get filesystem for directory name : " + dirPath.toUri(); - throw new RuntimeException(msg, e); - } - - } - } - /** * @return the listBucketingCtx */ diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index e8554f9ce5..141e871ba5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1645,9 +1645,9 @@ public Database getDatabaseCurrent() throws HiveException { * @return Partition object being loaded with data */ public Partition loadPartition(Path loadPath, Table tbl, Map partSpec, - LoadFileType loadFileType, boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir, - boolean isSrcLocal, boolean isAcidIUDoperation, boolean hasFollowingStatsTask, Long writeId, - int stmtId, boolean isInsertOverwrite) throws HiveException { + LoadFileType loadFileType, boolean inheritTableSpecs, boolean isSrcLocal, + boolean isAcidIUDoperation, boolean hasFollowingStatsTask, Long writeId, int stmtId, + boolean isInsertOverwrite) throws HiveException { Path tblDataLocationPath = tbl.getDataLocation(); boolean isMmTableWrite = AcidUtils.isInsertOnlyTable(tbl.getParameters()); assert tbl.getPath() != null : "null==getPath() for " + tbl.getTableName(); @@ -1765,17 +1765,6 @@ public Partition loadPartition(Path loadPath, Table tbl, Map par StatsSetupConst.clearColumnStatsState(newTPart.getParameters()); } - // recreate the partition if it existed before - if (isSkewedStoreAsSubdir) { - org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition(); - SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo(); - /* Construct list bucketing location mappings from sub-directory name. */ - Map, String> skewedColValueLocationMaps = constructListBucketingLocationMap( - newPartPath, skewedInfo); - /* Add list bucketing location mappings. */ - skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps); - newCreatedTpart.getSd().setSkewedInfo(skewedInfo); - } if (!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { StatsSetupConst.setBasicStatsState(newTPart.getParameters(), StatsSetupConst.FALSE); } @@ -2168,7 +2157,7 @@ public Void call() throws Exception { // load the partition Partition newPartition = loadPartition(partPath, tbl, fullPartSpec, loadFileType, - true, numLB > 0, false, isAcid, hasFollowingStatsTask, writeId, stmtId, + true, false, isAcid, hasFollowingStatsTask, writeId, stmtId, isInsertOverwrite); partitionsMap.put(fullPartSpec, newPartition); @@ -2253,17 +2242,15 @@ public Void call() throws Exception { * otherwise add files to table (KEEP_EXISTING, OVERWRITE_EXISTING) * @param isSrcLocal * If the source directory is LOCAL - * @param isSkewedStoreAsSubdir - * if list bucketing enabled * @param hasFollowingStatsTask * if there is any following stats task * @param isAcidIUDoperation true if this is an ACID based Insert [overwrite]/update/delete * @param writeId write ID allocated for the current load operation * @param stmtId statement ID of the current load statement */ - public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType, boolean isSrcLocal, - boolean isSkewedStoreAsSubdir, boolean isAcidIUDoperation, boolean hasFollowingStatsTask, - Long writeId, int stmtId, boolean isInsertOverwrite) throws HiveException { + public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType, + boolean isSrcLocal, boolean isAcidIUDoperation, boolean hasFollowingStatsTask, Long writeId, + int stmtId, boolean isInsertOverwrite) throws HiveException { List newFiles = null; Table tbl = getTable(tableName); assert tbl.getPath() != null : "null==getPath() for " + tbl.getTableName(); @@ -2326,20 +2313,6 @@ public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType StatsSetupConst.clearColumnStatsState(tbl.getParameters()); } - try { - if (isSkewedStoreAsSubdir) { - SkewedInfo skewedInfo = tbl.getSkewedInfo(); - // Construct list bucketing location mappings from sub-directory name. - Map, String> skewedColValueLocationMaps = constructListBucketingLocationMap( - tbl.getPath(), skewedInfo); - // Add list bucketing location mappings. - skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps); - } - } catch (IOException e) { - LOG.error(StringUtils.stringifyException(e)); - throw new HiveException(e); - } - EnvironmentContext environmentContext = null; if (hasFollowingStatsTask) { environmentContext = new EnvironmentContext(); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java index 9dbd869d57..f4f76649a2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java @@ -552,10 +552,6 @@ public void setLastAccessTime(int lastAccessTime) { tPartition.setLastAccessTime(lastAccessTime); } - public boolean isStoredAsSubDirectories() { - return tPartition.getSd().isStoredAsSubDirectories(); - } - public List> getSkewedColValues(){ return tPartition.getSd().getSkewedInfo().getSkewedColValues(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index ba16f842d2..1e4ebd1227 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -625,14 +625,6 @@ public void setSkewedInfo(SkewedInfo skewedInfo) throws HiveException { tTable.getSd().setSkewedInfo(skewedInfo); } - public boolean isStoredAsSubDirectories() { - return tTable.getSd().isStoredAsSubDirectories(); - } - - public void setStoredAsSubDirectories(boolean storedAsSubDirectories) throws HiveException { - tTable.getSd().setStoredAsSubDirectories(storedAsSubDirectories); - } - private boolean isField(String col) { for (FieldSchema field : getCols()) { if (field.getName().equals(col)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java index 44687ef471..8baac7c8e9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java @@ -382,9 +382,6 @@ private static void getStorageDescriptorInfo(StringBuilder tableInfo, formatOutput("Num Buckets:", String.valueOf(storageDesc.getNumBuckets()), tableInfo); formatOutput("Bucket Columns:", storageDesc.getBucketCols().toString(), tableInfo); formatOutput("Sort Columns:", storageDesc.getSortCols().toString(), tableInfo); - if (storageDesc.isStoredAsSubDirectories()) {// optional parameter - formatOutput("Stored As SubDirectories:", "Yes", tableInfo); - } if (null != storageDesc.getSkewedInfo()) { List skewedColNames = sortedList(storageDesc.getSkewedInfo().getSkewedColNames()); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 605bb09cab..ea6b0b47bd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1621,8 +1621,7 @@ public static MapWork createMergeTask(FileSinkDesc fsInputDesc, Path finalName, ArrayList inputDirstr = new ArrayList(1); // this will be populated by MergeFileWork.resolveDynamicPartitionStoredAsSubDirsMerge // in case of dynamic partitioning and list bucketing - if (!hasDynamicPartitions && - !GenMapRedUtils.isSkewedStoredAsDirs(fsInputDesc)) { + if (!hasDynamicPartitions) { inputDirs.add(inputDir); } inputDirstr.add(inputDir.toString()); @@ -1851,17 +1850,6 @@ private static ConditionalTask createCondTask(HiveConf conf, return cndTsk; } - /** - * check if it is skewed table and stored as dirs. - * - * @param fsInputDesc - * @return - */ - public static boolean isSkewedStoredAsDirs(FileSinkDesc fsInputDesc) { - return (fsInputDesc.getLbCtx() == null) ? false : fsInputDesc.getLbCtx() - .isSkewedStoredAsDir(); - } - public static Task findMoveTaskForFsopOutput( List> mvTasks, Path fsopFinalDir, boolean isMmFsop) { // find the move task diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 596eddedf5..b1ff80abf9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1839,17 +1839,15 @@ public boolean hasTransactionalInQuery() { * @param skewedColNames * @param skewedValues * @param skewedColValueLocationMaps - * @param isStoredAsSubDirectories * @return */ protected ListBucketingCtx constructListBucketingCtx(List skewedColNames, List> skewedValues, Map, String> skewedColValueLocationMaps, - boolean isStoredAsSubDirectories, HiveConf conf) { + HiveConf conf) { ListBucketingCtx lbCtx = new ListBucketingCtx(); lbCtx.setSkewedColNames(skewedColNames); lbCtx.setSkewedColValues(skewedValues); lbCtx.setLbLocationMap(skewedColValueLocationMaps); - lbCtx.setStoredAsSubDirectories(isStoredAsSubDirectories); lbCtx.setDefaultKey(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY); lbCtx.setDefaultDirName(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME); return lbCtx; @@ -1962,22 +1960,6 @@ protected void analyzeDDLSkewedValues(List> skewedValues, ASTNode c } } - /** - * process stored as directories - * - * @param child - * @return - */ - protected boolean analyzeStoredAdDirs(ASTNode child) { - boolean storedAsDirs = false; - if ((child.getChildCount() == 3) - && (((ASTNode) child.getChild(2)).getToken().getType() - == HiveParser.TOK_STOREDASDIRS)) { - storedAsDirs = true; - } - return storedAsDirs; - } - private static boolean getPartExprNodeDesc(ASTNode astNode, HiveConf conf, Map astExprNodeMap) throws SemanticException { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 37e98456f1..5ef24def24 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1503,8 +1503,6 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { Path oldTblPartLoc = null; List cols = null; ListBucketingCtx lbCtx = null; - boolean isListBucketed = false; - List listBucketColNames = null; if (table.isPartitioned()) { Partition part = db.getPartition(table, partSpec, false); @@ -1524,9 +1522,7 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { inputFormatClass = part.getInputFormatClass(); isArchived = ArchiveUtils.isArchived(part); lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(), - part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf); - isListBucketed = part.isStoredAsSubDirectories(); - listBucketColNames = part.getSkewedColNames(); + part.getSkewedColValueLocationMaps(), conf); } else { // input and output are the same oldTblPartLoc = table.getPath(); @@ -1535,9 +1531,7 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { bucketCols = table.getBucketCols(); inputFormatClass = table.getInputFormatClass(); lbCtx = constructListBucketingCtx(table.getSkewedColNames(), table.getSkewedColValues(), - table.getSkewedColValueLocationMaps(), table.isStoredAsSubDirectories(), conf); - isListBucketed = table.isStoredAsSubDirectories(); - listBucketColNames = table.getSkewedColNames(); + table.getSkewedColValueLocationMaps(), conf); } // throw a HiveException for non-rcfile. @@ -1570,14 +1564,6 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { throw new SemanticException(ErrorMsg.TRUNCATE_BUCKETED_COLUMN.getMsg(columnName)); } } - if (isListBucketed) { - for (String listBucketCol : listBucketColNames) { - if (listBucketCol.equalsIgnoreCase(columnName)) { - throw new SemanticException( - ErrorMsg.TRUNCATE_LIST_BUCKETED_COLUMN.getMsg(columnName)); - } - } - } } truncateTblDesc.setColumnIndexes(new ArrayList(columnIndexes)); @@ -2025,7 +2011,7 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, oldTblPartLoc = partPath; lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(), - part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf); + part.getSkewedColValueLocationMaps(), conf); } } else { inputFormatClass = tblObj.getInputFormatClass(); @@ -2036,7 +2022,7 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, newTblPartLoc = tblObj.getPath(); lbCtx = constructListBucketingCtx(tblObj.getSkewedColNames(), tblObj.getSkewedColValues(), - tblObj.getSkewedColValueLocationMaps(), tblObj.isStoredAsSubDirectories(), conf); + tblObj.getSkewedColValueLocationMaps(), conf); } // throw a HiveException for other than rcfile and orcfile. @@ -4074,7 +4060,6 @@ private void analyzeAltertableSkewedby(String[] qualified, ASTNode ast) throws S /* Convert a skewed table to non-skewed table. */ AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, true, new ArrayList(), new ArrayList>()); - alterTblDesc.setStoredAsSubDirectories(false); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc))); } else { @@ -4082,9 +4067,6 @@ private void analyzeAltertableSkewedby(String[] qualified, ASTNode ast) throws S case HiveParser.TOK_TABLESKEWED: handleAlterTableSkewedBy(ast, tableName, tab); break; - case HiveParser.TOK_STOREDASDIRS: - handleAlterTableDisableStoredAsDirs(tableName, tab); - break; default: assert false; } @@ -4092,29 +4074,7 @@ private void analyzeAltertableSkewedby(String[] qualified, ASTNode ast) throws S } /** - * Handle alter table not stored as directories - * - * @param tableName - * @param tab - * @throws SemanticException - */ - private void handleAlterTableDisableStoredAsDirs(String tableName, Table tab) - throws SemanticException { - List skewedColNames = tab.getSkewedColNames(); - List> skewedColValues = tab.getSkewedColValues(); - if ((skewedColNames == null) || (skewedColNames.size() == 0) || (skewedColValues == null) - || (skewedColValues.size() == 0)) { - throw new SemanticException(ErrorMsg.ALTER_TBL_STOREDASDIR_NOT_SKEWED.getMsg(tableName)); - } - AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, false, - skewedColNames, skewedColValues); - alterTblDesc.setStoredAsSubDirectories(false); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - alterTblDesc))); - } - - /** - * Process "alter table skewed by .. on .. stored as directories + * Process "alter table skewed by .. on .. * @param ast * @param tableName * @param tab @@ -4129,13 +4089,9 @@ private void handleAlterTableSkewedBy(ASTNode ast, String tableName, Table tab) skewedColNames = analyzeSkewedTablDDLColNames(skewedColNames, skewedNode); /* skewed value. */ analyzeDDLSkewedValues(skewedValues, skewedNode); - // stored as directories - boolean storedAsDirs = analyzeStoredAdDirs(skewedNode); - AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, false, skewedColNames, skewedValues); - alterTblDesc.setStoredAsSubDirectories(storedAsDirs); /** * Validate information about skewed table */ @@ -4145,51 +4101,6 @@ private void handleAlterTableSkewedBy(ASTNode ast, String tableName, Table tab) alterTblDesc))); } - /** - * Analyze skewed column names - * - * @param skewedColNames - * @param child - * @return - * @throws SemanticException - */ - private List analyzeAlterTableSkewedColNames(List skewedColNames, - ASTNode child) throws SemanticException { - Tree nNode = child.getChild(0); - if (nNode == null) { - throw new SemanticException(ErrorMsg.SKEWED_TABLE_NO_COLUMN_NAME.getMsg()); - } else { - ASTNode nAstNode = (ASTNode) nNode; - if (nAstNode.getToken().getType() != HiveParser.TOK_TABCOLNAME) { - throw new SemanticException(ErrorMsg.SKEWED_TABLE_NO_COLUMN_NAME.getMsg()); - } else { - skewedColNames = getColumnNames(nAstNode); - } - } - return skewedColNames; - } - - /** - * Given a ASTNode, return list of values. - * - * use case: - * create table xyz list bucketed (col1) with skew (1,2,5) - * AST Node is for (1,2,5) - * - * @param ast - * @return - */ - private List getColumnValues(ASTNode ast) { - List colList = new ArrayList(); - int numCh = ast.getChildCount(); - for (int i = 0; i < numCh; i++) { - ASTNode child = (ASTNode) ast.getChild(i); - colList.add(stripQuotes(child.getText()).toLowerCase()); - } - return colList; - } - - /** * Analyze alter table's skewed location * diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 8726974186..30c65d604b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -339,7 +339,6 @@ TOK_TABCOLVALUES; TOK_SKEWED_LOCATIONS; TOK_SKEWED_LOCATION_LIST; TOK_SKEWED_LOCATION_MAP; -TOK_STOREDASDIRS; TOK_PARTITIONINGSPEC; TOK_PTBLFUNCTION; TOK_WINDOWDEF; @@ -995,13 +994,6 @@ rewriteDisabled -> ^(TOK_REWRITE_DISABLED) ; -storedAsDirs -@init { pushMsg("stored as directories", state); } -@after { popMsg(state); } - : KW_STORED KW_AS KW_DIRECTORIES - -> ^(TOK_STOREDASDIRS) - ; - orReplace @init { pushMsg("or replace clause", state); } @after { popMsg(state); } @@ -1431,9 +1423,6 @@ alterStatementSuffixSkewedby | KW_NOT KW_SKEWED ->^(TOK_ALTERTABLE_SKEWED) - | - KW_NOT storedAsDirs - ->^(TOK_ALTERTABLE_SKEWED storedAsDirs) ; alterStatementSuffixExchangePartition @@ -1992,8 +1981,8 @@ tableSkewed @init { pushMsg("table skewed specification", state); } @after { popMsg(state); } : - KW_SKEWED KW_BY LPAREN skewedCols=columnNameList RPAREN KW_ON LPAREN (skewedValues=skewedValueElement) RPAREN ((storedAsDirs) => storedAsDirs)? - -> ^(TOK_TABLESKEWED $skewedCols $skewedValues storedAsDirs?) + KW_SKEWED KW_BY LPAREN skewedCols=columnNameList RPAREN KW_ON LPAREN (skewedValues=skewedValueElement) RPAREN + -> ^(TOK_TABLESKEWED $skewedCols $skewedValues) ; rowFormat diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 189975ed2e..3bcd7bda25 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -316,9 +316,6 @@ private void analyzeLoad(ASTNode ast) throws SemanticException { throw new SemanticException(ErrorMsg.LOAD_INTO_NON_NATIVE.getMsg()); } - if(ts.tableHandle.isStoredAsSubDirectories()) { - throw new SemanticException(ErrorMsg.LOAD_INTO_STORED_AS_DIR.getMsg()); - } List parts = ts.tableHandle.getPartitionKeys(); if ((parts != null && parts.size() > 0) && (ts.partSpec == null || ts.partSpec.size() == 0)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index ff952b6950..312c485032 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7248,9 +7248,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) currentTableId = destTableId; destTableId++; - lbCtx = constructListBucketingCtx(dest_tab.getSkewedColNames(), - dest_tab.getSkewedColValues(), dest_tab.getSkewedColValueLocationMaps(), - dest_tab.isStoredAsSubDirectories(), conf); + lbCtx = constructListBucketingCtx(dest_tab.getSkewedColNames(), dest_tab.getSkewedColValues(), + dest_tab.getSkewedColValueLocationMaps(), conf); // Create the work for moving the table // NOTE: specify Dynamic partitions in dest_tab for WriteEntity @@ -7346,8 +7345,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) destTableId++; lbCtx = constructListBucketingCtx(dest_part.getSkewedColNames(), - dest_part.getSkewedColValues(), dest_part.getSkewedColValueLocationMaps(), - dest_part.isStoredAsSubDirectories(), conf); + dest_part.getSkewedColValues(), dest_part.getSkewedColValueLocationMaps(), conf); AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID; if (destTableIsFullAcid) { acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest); @@ -7730,7 +7728,6 @@ private FileSinkDesc createFileSinkDesc(String dest, TableDesc table_desc, /* Set List Bucketing context. */ if (lbCtx != null) { lbCtx.processRowSkewedIndex(fsRS); - lbCtx.calculateSkewedValueSubDirList(); } fileSinkDesc.setLbCtx(lbCtx); @@ -12890,7 +12887,6 @@ ASTNode analyzeCreateTable( List skewedColNames = new ArrayList(); List> skewedValues = new ArrayList>(); Map, String> listBucketColValuesMapping = new HashMap, String>(); - boolean storedAsDirs = false; boolean isUserStorageFormat = false; RowFormatParams rowFormatParams = new RowFormatParams(); @@ -13020,9 +13016,6 @@ ASTNode analyzeCreateTable( skewedColNames = analyzeSkewedTablDDLColNames(skewedColNames, child); // skewed value analyzeDDLSkewedValues(skewedValues, child); - // stored as directories - storedAsDirs = analyzeStoredAdDirs(child); - break; default: throw new AssertionError("Unknown token: " + child.getToken()); @@ -13101,7 +13094,6 @@ ASTNode analyzeCreateTable( storageFormat.getStorageHandler(), storageFormat.getSerdeProps(), tblProps, ifNotExists, skewedColNames, skewedValues, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints); - crtTblDesc.setStoredAsSubDirectories(storedAsDirs); crtTblDesc.setNullFormat(rowFormatParams.nullFormat); crtTblDesc.validate(conf); @@ -13204,7 +13196,6 @@ ASTNode analyzeCreateTable( skewedColNames, skewedValues, true, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints); tableDesc.setMaterialization(isMaterialization); - tableDesc.setStoredAsSubDirectories(storedAsDirs); tableDesc.setNullFormat(rowFormatParams.nullFormat); qb.setTableDesc(tableDesc); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java index 512f1ff3da..cd2bfed416 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -1113,7 +1113,6 @@ private boolean handleCardinalityViolation(StringBuilder rewrittenQueryStr, ASTN table.setDataLocation(Warehouse.getDnsPath(new Path(SessionState.get().getTempTableSpace(), tableName), conf)); table.getTTable().setTemporary(true); - table.setStoredAsSubDirectories(false); table.setInputFormatClass(format.getInputFormat()); table.setOutputFormatClass(format.getOutputFormat()); db.createTable(table, true); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java index 0b04c0ce85..ba03e88630 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java @@ -121,7 +121,6 @@ ProtectModeType protectModeType; Map, String> skewedLocations; boolean isTurnOffSkewed = false; - boolean isStoredAsSubDirectories = false; List skewedColNames; List> skewedColValues; Table tableForSkewedColValidation; @@ -888,20 +887,6 @@ public void setTable(Table table) { this.tableForSkewedColValidation = table; } - /** - * @return the isStoredAsSubDirectories - */ - public boolean isStoredAsSubDirectories() { - return isStoredAsSubDirectories; - } - - /** - * @param isStoredAsSubDirectories the isStoredAsSubDirectories to set - */ - public void setStoredAsSubDirectories(boolean isStoredAsSubDirectories) { - this.isStoredAsSubDirectories = isStoredAsSubDirectories; - } - /** * @param isDropIfExists the isDropIfExists to set */ diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java index 04292787a8..658451efb6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java @@ -92,7 +92,6 @@ boolean ifNotExists; List skewedColNames; List> skewedColValues; - boolean isStoredAsSubDirectories = false; boolean isTemporary = false; private boolean isMaterialization = false; private boolean replaceMode = false; @@ -605,20 +604,6 @@ public void validate(HiveConf conf) this.getSkewedColValues()); } - /** - * @return the isStoredAsSubDirectories - */ - public boolean isStoredAsSubDirectories() { - return isStoredAsSubDirectories; - } - - /** - * @param isStoredAsSubDirectories the isStoredAsSubDirectories to set - */ - public void setStoredAsSubDirectories(boolean isStoredAsSubDirectories) { - this.isStoredAsSubDirectories = isStoredAsSubDirectories; - } - /** * @return the nullFormat */ @@ -806,8 +791,6 @@ public Table toTable(HiveConf conf) throws HiveException { tbl.getTTable().setTemporary(isTemporary()); - tbl.setStoredAsSubDirectories(isStoredAsSubDirectories()); - tbl.setInputFormatClass(getInputFormat()); tbl.setOutputFormatClass(getOutputFormat()); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java index ef7325fe2c..ed80240c65 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java @@ -85,7 +85,6 @@ public ImportTableDesc(String dbName, Table table) throws Exception { null, null, null); - this.createTblDesc.setStoredAsSubDirectories(table.getSd().isStoredAsSubDirectories()); break; case VIEW: String[] qualViewName = { dbName, table.getTableName() }; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java index 5f8cf54d57..c326b590df 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java @@ -41,14 +41,11 @@ private List> skewedColValues; private Map, String> lbLocationMap; private List rowSkewedIndex; - private boolean isStoredAsSubDirectories; private String defaultKey; private String defaultDirName; - private List skewedValuesDirNames; public ListBucketingCtx() { rowSkewedIndex = new ArrayList(); - skewedValuesDirNames = new ArrayList(); } /** @@ -118,41 +115,6 @@ public void processRowSkewedIndex(RowSchema rowSch) { } } - /** - * Calculate skewed value subdirectory directory which is used in - * FileSinkOperator.java createKeyForStatsPublisher() - * For example, create table test skewed by (key, value) on (('484','val_484') - * stored as DIRECTORIES; - * after the method, skewedValuesDirNames will contain 2 elements: - * key=484/value=val_484 - * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME - */ - public void calculateSkewedValueSubDirList() { - if (isSkewedStoredAsDir()) { - for (List value : this.skewedColValues) { - skewedValuesDirNames.add(FileUtils.makeListBucketingDirName(this.skewedColNames, value)); - } - // creat default dir - skewedValuesDirNames.add(FileUtils.makeDefaultListBucketingDirName( - this.skewedColNames, - ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME)); - } - } - - /** - * @return the isStoredAsSubDirectories - */ - public boolean isStoredAsSubDirectories() { - return isStoredAsSubDirectories; - } - - /** - * @param isStoredAsSubDirectories the isStoredAsSubDirectories to set - */ - public void setStoredAsSubDirectories(boolean isStoredAsSubDirectories) { - this.isStoredAsSubDirectories = isStoredAsSubDirectories; - } - /** * @return the defaultKey */ @@ -181,46 +143,16 @@ public void setDefaultDirName(String defaultDirName) { this.defaultDirName = defaultDirName; } - /** - * check if list bucketing is enabled. - * - * @param ctx - * @return - */ - public boolean isSkewedStoredAsDir() { - return (this.getSkewedColNames() != null) - && (this.getSkewedColNames().size() > 0) - && (this.getSkewedColValues() != null) - && (this.getSkewedColValues().size() > 0) - && (this.isStoredAsSubDirectories()); - } - /** * Calculate list bucketing level. * - * 0: not list bucketing - * int: no. of skewed columns + * no list bucketing, it's been removed with HIVE-17852 * * @param ctx * @return */ - public int calculateListBucketingLevel() { - int lbLevel = isSkewedStoredAsDir() ? this.getSkewedColNames().size() : 0; - return lbLevel; - } - - /** - * @return the skewedValuesDirNames - */ - public List getSkewedValuesDirNames() { - return skewedValuesDirNames; - } - - /** - * @param skewedValuesDirNames the skewedValuesDirNames to set - */ - public void setSkewedValuesDirNames(List skewedValuesDirNames) { - this.skewedValuesDirNames = skewedValuesDirNames; + public int calculateListBucketingLevel() { + return 0; } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/SkewedColumnPositionPair.java ql/src/java/org/apache/hadoop/hive/ql/plan/SkewedColumnPositionPair.java index 8bb40abc8d..eb2ba492ad 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/SkewedColumnPositionPair.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/SkewedColumnPositionPair.java @@ -25,14 +25,11 @@ * create table list_bucketing_static_part (key String, value String) * partitioned by (ds String, hr String) * skewed by (value) on ('val_466','val_287','val_82') - * stored as DIRECTORIES * STORED AS RCFILE; * * Skewed column is "value". * 1. It's position in table column is 1. * 2. It's position in skewed column list is 0. - * - * This information will be used in {@FileSinkOperator} generateListBucketingDirName */ public class SkewedColumnPositionPair { private int tblColPosition; diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java index b698c84080..3c680f5dcc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java @@ -44,24 +44,20 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.StringableMap; import org.apache.hadoop.hive.common.ValidCompactorWriteIdList; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.CompactionType; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.txn.CompactionInfo; import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.exec.DDLTask; @@ -75,7 +71,6 @@ import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; -import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -442,24 +437,6 @@ private String buildMmCompactionCtQuery( query.append("INTO ").append(t.getSd().getNumBuckets()).append(" BUCKETS"); } - // Stored as directories. We don't care about the skew otherwise. - if (t.getSd().isStoredAsSubDirectories()) { - SkewedInfo skewedInfo = t.getSd().getSkewedInfo(); - if (skewedInfo != null && !skewedInfo.getSkewedColNames().isEmpty()) { - query.append(" SKEWED BY (").append( - StringUtils.join(", ", skewedInfo.getSkewedColNames())).append(") ON "); - isFirst = true; - for (List colValues : skewedInfo.getSkewedColValues()) { - if (!isFirst) { - query.append(", "); - } - isFirst = false; - query.append("('").append(StringUtils.join("','", colValues)).append("')"); - } - query.append(") STORED AS DIRECTORIES"); - } - } - SerDeInfo serdeInfo = sd.getSerdeInfo(); Map serdeParams = serdeInfo.getParameters(); query.append(" ROW FORMAT SERDE '").append(HiveStringUtils.escapeHiveCommand( diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index e108684660..bf366d9e52 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -142,7 +142,7 @@ db.createTable(src, cols, null, TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class); db.loadTable(hadoopDataFile[i], src, LoadFileType.KEEP_EXISTING, - true, false, false, false, null, 0, false); + true, false, false, null, 0, false); i++; } diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index a24b6423ba..bd0a01bcf0 100755 --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -165,7 +165,6 @@ public void testTable() throws Throwable { tbl.setSerdeParam(serdeConstants.FIELD_DELIM, "1"); tbl.setSerializationLib(LazySimpleSerDe.class.getName()); - tbl.setStoredAsSubDirectories(false); tbl.setRewriteEnabled(false); @@ -226,7 +225,6 @@ public void testThriftTable() throws Throwable { tbl.setSerdeParam(serdeConstants.SERIALIZATION_CLASS, Complex.class.getName()); tbl.setSerdeParam(serdeConstants.SERIALIZATION_FORMAT, TBinaryProtocol.class .getName()); - tbl.setStoredAsSubDirectories(false); tbl.setRewriteEnabled(false); diff --git ql/src/test/queries/clientnegative/load_stored_as_dirs.q ql/src/test/queries/clientnegative/load_stored_as_dirs.q deleted file mode 100644 index 69e23099b9..0000000000 --- ql/src/test/queries/clientnegative/load_stored_as_dirs.q +++ /dev/null @@ -1,5 +0,0 @@ --- Load data can't work with table with stored as directories -CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES; - -LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE stored_as_dirs_multiple; diff --git ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q deleted file mode 100644 index 70cbce865a..0000000000 --- ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q +++ /dev/null @@ -1,14 +0,0 @@ ---! qt:dataset:src -set mapred.input.dir.recursive=true; - --- Tests truncating a column on which a table is list bucketed - -CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE; - -ALTER TABLE test_tab -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES; - -INSERT OVERWRITE TABLE test_tab SELECT * FROM src; - -TRUNCATE TABLE test_tab COLUMNS (key); diff --git ql/src/test/queries/clientpositive/create_alter_list_bucketing_table1.q ql/src/test/queries/clientpositive/create_alter_list_bucketing_table1.q deleted file mode 100644 index bf89e8feb7..0000000000 --- ql/src/test/queries/clientpositive/create_alter_list_bucketing_table1.q +++ /dev/null @@ -1,38 +0,0 @@ --- Test stored as directories --- it covers a few cases - --- 1. create a table with stored as directories -CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES; -describe formatted stored_as_dirs_multiple; - --- 2. turn off stored as directories but table is still a skewed table -alter table stored_as_dirs_multiple not stored as DIRECTORIES; -describe formatted stored_as_dirs_multiple; - --- 3. turn off skewed -alter table stored_as_dirs_multiple not skewed; -describe formatted stored_as_dirs_multiple; - --- 4. alter a table to stored as directories -CREATE TABLE stored_as_dirs_single (key STRING, value STRING); -alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES; -describe formatted stored_as_dirs_single; - --- 5. turn off skewed should turn off stored as directories too -alter table stored_as_dirs_single not skewed; -describe formatted stored_as_dirs_single; - --- 6. turn on stored as directories again -alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES; -describe formatted stored_as_dirs_single; - --- 7. create table like -create table stored_as_dirs_single_like like stored_as_dirs_single; -describe formatted stored_as_dirs_single_like; - --- cleanup -drop table stored_as_dirs_single; -drop table stored_as_dirs_multiple; diff --git ql/src/test/queries/clientpositive/infer_bucket_sort_list_bucket.q ql/src/test/queries/clientpositive/infer_bucket_sort_list_bucket.q index 547946822a..bcc66b98b2 100644 --- ql/src/test/queries/clientpositive/infer_bucket_sort_list_bucket.q +++ ql/src/test/queries/clientpositive/infer_bucket_sort_list_bucket.q @@ -11,8 +11,7 @@ set mapred.input.dir.recursive=true; -- create a skewed table CREATE TABLE list_bucketing_table (key STRING, value STRING) PARTITIONED BY (part STRING) -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES; +SKEWED BY (key) ON ("484"); -- Tests group by, the output should neither be bucketed nor sorted @@ -24,8 +23,7 @@ DESC FORMATTED list_bucketing_table PARTITION (part = '1'); -- create a table skewed on a key which doesnt exist in the data CREATE TABLE list_bucketing_table2 (key STRING, value STRING) PARTITIONED BY (part STRING) -SKEWED BY (key) ON ("abc") -STORED AS DIRECTORIES; +SKEWED BY (key) ON ("abc"); -- should not be bucketed or sorted INSERT OVERWRITE TABLE list_bucketing_table2 PARTITION (part = '1') diff --git ql/src/test/queries/clientpositive/lb_fs_stats.q ql/src/test/queries/clientpositive/lb_fs_stats.q index 08baae74bd..850d8e6fd5 100644 --- ql/src/test/queries/clientpositive/lb_fs_stats.q +++ ql/src/test/queries/clientpositive/lb_fs_stats.q @@ -11,7 +11,7 @@ set hive.stats.dbclass=fs; CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE; -ALTER TABLE test_tab SKEWED BY (key) ON ("484") STORED AS DIRECTORIES; +ALTER TABLE test_tab SKEWED BY (key) ON ("484"); INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_1.q ql/src/test/queries/clientpositive/list_bucket_dml_1.q index 7196d6adc1..f555ec1a51 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_1.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_1.q @@ -14,9 +14,7 @@ set mapred.input.dir.recursive=true; -- create a skewed table create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) -skewed by (key) on ("484") -stored as DIRECTORIES -; +skewed by (key) on ("484"); -- list bucketing DML explain extended diff --git ql/src/test/queries/clientpositive/list_bucket_dml_10.q ql/src/test/queries/clientpositive/list_bucket_dml_10.q index 60bad7fac2..f3437c4f67 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_10.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_10.q @@ -18,7 +18,6 @@ set mapred.input.dir.recursive=true; create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_11.q ql/src/test/queries/clientpositive/list_bucket_dml_11.q index 7ebf856fbc..bc33cb22f6 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_11.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_11.q @@ -14,7 +14,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (value) on ('val_466','val_287','val_82') - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_12.q ql/src/test/queries/clientpositive/list_bucket_dml_12.q index daaffbf04f..89e265c8d2 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_12.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_12.q @@ -13,7 +13,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML diff --git ql/src/test/queries/clientpositive/list_bucket_dml_13.q ql/src/test/queries/clientpositive/list_bucket_dml_13.q index 091cf0cb58..fcd045f1b3 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_13.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_13.q @@ -13,7 +13,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML diff --git ql/src/test/queries/clientpositive/list_bucket_dml_14.q ql/src/test/queries/clientpositive/list_bucket_dml_14.q index a0f9c2c376..f14a033f10 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_14.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_14.q @@ -12,9 +12,7 @@ set mapred.input.dir.recursive=true; -- create a skewed table create table list_bucketing (key String, value String) -skewed by (key) on ("484") -stored as DIRECTORIES -; +skewed by (key) on ("484"); -- list bucketing DML explain extended diff --git ql/src/test/queries/clientpositive/list_bucket_dml_2.q ql/src/test/queries/clientpositive/list_bucket_dml_2.q index 948eabe4ad..c047b90928 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_2.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_2.q @@ -27,7 +27,6 @@ set hive.stats.reliable=true; create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_3.q ql/src/test/queries/clientpositive/list_bucket_dml_3.q index e66861ec39..c84d20fc46 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_3.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_3.q @@ -12,7 +12,7 @@ set mapred.input.dir.recursive=true; -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23) -- create a skewed table -create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") stored as DIRECTORIES; +create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484"); -- list bucketing DML explain extended diff --git ql/src/test/queries/clientpositive/list_bucket_dml_4.q ql/src/test/queries/clientpositive/list_bucket_dml_4.q index c19ebe7417..ea5c015e11 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_4.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_4.q @@ -30,7 +30,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_5.q ql/src/test/queries/clientpositive/list_bucket_dml_5.q index bd6a5cb728..ac7c7e1b19 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_5.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_5.q @@ -15,8 +15,7 @@ set mapred.input.dir.recursive=true; -- create a skewed table create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) -stored as DIRECTORIES; +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')); -- list bucketing DML explain extended diff --git ql/src/test/queries/clientpositive/list_bucket_dml_6.q ql/src/test/queries/clientpositive/list_bucket_dml_6.q index 6cb29cb2a3..be1b1719e1 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_6.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_6.q @@ -53,7 +53,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_7.q ql/src/test/queries/clientpositive/list_bucket_dml_7.q index 2c96407259..e83ecfdb33 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_7.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_7.q @@ -28,7 +28,6 @@ set hive.merge.rcfile.block.level=true; CREATE TABLE list_bucketing_dynamic_part (key String, value STRING) PARTITIONED BY (ds string, hr string) skewed by (key) on ('484') - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_8.q ql/src/test/queries/clientpositive/list_bucket_dml_8.q index f215852dcf..de9e14edef 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_8.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_8.q @@ -54,7 +54,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_dml_9.q ql/src/test/queries/clientpositive/list_bucket_dml_9.q index b9de4307a4..0caeccdd78 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_9.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_9.q @@ -30,7 +30,6 @@ set hive.merge.mapredfiles=false; create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','103') - stored as DIRECTORIES STORED AS RCFILE; -- list bucketing DML without merge. use bucketize to generate a few small files. diff --git ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q index 9e377d5763..988cc2c382 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q +++ ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q @@ -22,8 +22,7 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; -- create a skewed table create table fact_daily (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES; +skewed by (key, value) on (('484','val_484'),('238','val_238')); insert overwrite table fact_daily partition (ds = '1', hr = '4') select key, value from src; diff --git ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q index 9af9132055..53e5c6abf2 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q +++ ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q @@ -20,8 +20,7 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; -- create a skewed table create table fact_daily (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES; +skewed by (key, value) on (('484','val_484'),('238','val_238')); insert overwrite table fact_daily partition (ds = '1', hr = '4') select key, value from src; diff --git ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q index b68ec64ca5..81d9e31f34 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q +++ ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q @@ -28,13 +28,13 @@ select key, value from src; describe formatted fact_daily PARTITION (ds = '1', hr='1'); -- partition. skewed value is 484/238 -alter table fact_daily skewed by (key, value) on (('484','val_484'),('238','val_238')) stored as DIRECTORIES; +alter table fact_daily skewed by (key, value) on (('484','val_484'),('238','val_238')); insert overwrite table fact_daily partition (ds = '1', hr = '2') select key, value from src; describe formatted fact_daily PARTITION (ds = '1', hr='2'); -- another partition. skewed value is 327 -alter table fact_daily skewed by (key, value) on (('327','val_327')) stored as DIRECTORIES; +alter table fact_daily skewed by (key, value) on (('327','val_327')); insert overwrite table fact_daily partition (ds = '1', hr = '3') select key, value from src; describe formatted fact_daily PARTITION (ds = '1', hr='3'); diff --git ql/src/test/queries/clientpositive/mm_all.q ql/src/test/queries/clientpositive/mm_all.q index ceef0c072d..e1bb20a71e 100644 --- ql/src/test/queries/clientpositive/mm_all.q +++ ql/src/test/queries/clientpositive/mm_all.q @@ -121,7 +121,7 @@ drop table partunion_mm; set mapreduce.input.fileinputformat.input.dir.recursive=true; create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only"); +tblproperties ("transactional"="true", "transactional_properties"="insert_only"); insert into table skew_mm select key, key, key from intermediate; @@ -131,7 +131,7 @@ drop table skew_mm; create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only"); +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only"); insert into table skew_dp_union_mm partition (k3) select key as i, key as j, key as k, key as l from intermediate diff --git ql/src/test/queries/clientpositive/show_create_table_db_table.q ql/src/test/queries/clientpositive/show_create_table_db_table.q index 35ac5bf73e..f98ea8abc9 100644 --- ql/src/test/queries/clientpositive/show_create_table_db_table.q +++ ql/src/test/queries/clientpositive/show_create_table_db_table.q @@ -4,7 +4,7 @@ CREATE DATABASE tmp_feng comment 'for show create table test'; SHOW DATABASES; CREATE TABLE tmp_feng.tmp_showcrt1(key string, value int); CREATE TABLE tmp_feng.tmp_showcrt2(key string, value int) skewed by (key) on ('1','2'); -CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') stored as directories; +CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2'); USE default; SHOW CREATE TABLE tmp_feng.tmp_showcrt1; SHOW CREATE TABLE tmp_feng.tmp_showcrt2; diff --git ql/src/test/queries/clientpositive/stats_list_bucket.q ql/src/test/queries/clientpositive/stats_list_bucket.q index c4339d03ce..767ab78d61 100644 --- ql/src/test/queries/clientpositive/stats_list_bucket.q +++ ql/src/test/queries/clientpositive/stats_list_bucket.q @@ -10,7 +10,6 @@ create table stats_list_bucket ( c2 string ) partitioned by (ds string, hr string) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile; @@ -30,7 +29,6 @@ create table stats_list_bucket_1 ( c2 string ) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile; insert overwrite table stats_list_bucket_1 diff --git ql/src/test/queries/clientpositive/truncate_column_list_bucket.q ql/src/test/queries/clientpositive/truncate_column_list_bucket.q index 44f580652f..1ed55ea304 100644 --- ql/src/test/queries/clientpositive/truncate_column_list_bucket.q +++ ql/src/test/queries/clientpositive/truncate_column_list_bucket.q @@ -13,8 +13,7 @@ set mapred.input.dir.recursive=true; CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE; ALTER TABLE test_tab -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES; +SKEWED BY (key) ON ("484"); INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src; diff --git ql/src/test/results/clientnegative/load_stored_as_dirs.q.out ql/src/test/results/clientnegative/load_stored_as_dirs.q.out deleted file mode 100644 index bcc50e59b8..0000000000 --- ql/src/test/results/clientnegative/load_stored_as_dirs.q.out +++ /dev/null @@ -1,11 +0,0 @@ -PREHOOK: query: CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@stored_as_dirs_multiple -FAILED: SemanticException [Error 10195]: A stored-as-directories table cannot be used as target for LOAD diff --git ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out deleted file mode 100644 index 3b6ddde221..0000000000 --- ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out +++ /dev/null @@ -1,31 +0,0 @@ -PREHOOK: query: CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test_tab -POSTHOOK: query: CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test_tab -PREHOOK: query: ALTER TABLE test_tab -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@test_tab -PREHOOK: Output: default@test_tab -POSTHOOK: query: ALTER TABLE test_tab -SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@test_tab -POSTHOOK: Output: default@test_tab -PREHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@test_tab -POSTHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@test_tab -POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: A column on which a partition/table is list bucketed cannot be truncated. key diff --git ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out deleted file mode 100644 index 593ae83899..0000000000 --- ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out +++ /dev/null @@ -1,352 +0,0 @@ -PREHOOK: query: CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: CREATE TABLE if not exists stored_as_dirs_multiple (col1 STRING, col2 int, col3 STRING) -SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) stored as DIRECTORIES -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@stored_as_dirs_multiple -PREHOOK: query: describe formatted stored_as_dirs_multiple -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: query: describe formatted stored_as_dirs_multiple -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_multiple -# col_name data_type comment -col1 string -col2 int -col3 string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}} - bucketing_version 2 - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Stored As SubDirectories: Yes -Skewed Columns: [col1, col2] -Skewed Values: [[s1, 1], [s13, 13], [s3, 3], [s78, 78]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: alter table stored_as_dirs_multiple not stored as DIRECTORIES -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_multiple -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: alter table stored_as_dirs_multiple not stored as DIRECTORIES -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: Output: default@stored_as_dirs_multiple -PREHOOK: query: describe formatted stored_as_dirs_multiple -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: query: describe formatted stored_as_dirs_multiple -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_multiple -# col_name data_type comment -col1 string -col2 int -col3 string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Skewed Columns: [col1, col2] -Skewed Values: [[s1, 1], [s13, 13], [s3, 3], [s78, 78]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: alter table stored_as_dirs_multiple not skewed -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_multiple -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: alter table stored_as_dirs_multiple not skewed -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: Output: default@stored_as_dirs_multiple -PREHOOK: query: describe formatted stored_as_dirs_multiple -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: query: describe formatted stored_as_dirs_multiple -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_multiple -# col_name data_type comment -col1 string -col2 int -col3 string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: CREATE TABLE stored_as_dirs_single (key STRING, value STRING) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: CREATE TABLE stored_as_dirs_single (key STRING, value STRING) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_single -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_single -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: describe formatted stored_as_dirs_single -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_single -POSTHOOK: query: describe formatted stored_as_dirs_single -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_single -# col_name data_type comment -key string -value string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Stored As SubDirectories: Yes -Skewed Columns: [key] -Skewed Values: [[1], [5], [6]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: alter table stored_as_dirs_single not skewed -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_single -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: alter table stored_as_dirs_single not skewed -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_single -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: describe formatted stored_as_dirs_single -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_single -POSTHOOK: query: describe formatted stored_as_dirs_single -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_single -# col_name data_type comment -key string -value string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES -PREHOOK: type: ALTERTABLE_SKEWED -PREHOOK: Input: default@stored_as_dirs_single -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: alter table stored_as_dirs_single SKEWED BY (key) ON ('1','5','6') -stored as DIRECTORIES -POSTHOOK: type: ALTERTABLE_SKEWED -POSTHOOK: Input: default@stored_as_dirs_single -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: describe formatted stored_as_dirs_single -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_single -POSTHOOK: query: describe formatted stored_as_dirs_single -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_single -# col_name data_type comment -key string -value string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - bucketing_version 2 -#### A masked pattern was here #### - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Stored As SubDirectories: Yes -Skewed Columns: [key] -Skewed Values: [[1], [5], [6]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: create table stored_as_dirs_single_like like stored_as_dirs_single -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@stored_as_dirs_single_like -POSTHOOK: query: create table stored_as_dirs_single_like like stored_as_dirs_single -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@stored_as_dirs_single_like -PREHOOK: query: describe formatted stored_as_dirs_single_like -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@stored_as_dirs_single_like -POSTHOOK: query: describe formatted stored_as_dirs_single_like -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@stored_as_dirs_single_like -# col_name data_type comment -key string -value string - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: MANAGED_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 0 - numRows 0 - rawDataSize 0 - totalSize 0 -#### A masked pattern was here #### - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat -Compressed: No -Num Buckets: -1 -Bucket Columns: [] -Sort Columns: [] -Stored As SubDirectories: Yes -Skewed Columns: [key] -Skewed Values: [[1], [5], [6]] -Storage Desc Params: - serialization.format 1 -PREHOOK: query: drop table stored_as_dirs_single -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@stored_as_dirs_single -PREHOOK: Output: default@stored_as_dirs_single -POSTHOOK: query: drop table stored_as_dirs_single -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@stored_as_dirs_single -POSTHOOK: Output: default@stored_as_dirs_single -PREHOOK: query: drop table stored_as_dirs_multiple -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@stored_as_dirs_multiple -PREHOOK: Output: default@stored_as_dirs_multiple -POSTHOOK: query: drop table stored_as_dirs_multiple -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@stored_as_dirs_multiple -POSTHOOK: Output: default@stored_as_dirs_multiple diff --git ql/src/test/results/clientpositive/infer_bucket_sort_list_bucket.q.out ql/src/test/results/clientpositive/infer_bucket_sort_list_bucket.q.out index c701de9347..7cf9943263 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_list_bucket.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_list_bucket.q.out @@ -1,14 +1,12 @@ PREHOOK: query: CREATE TABLE list_bucketing_table (key STRING, value STRING) PARTITIONED BY (part STRING) SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_table POSTHOOK: query: CREATE TABLE list_bucketing_table (key STRING, value STRING) PARTITIONED BY (part STRING) SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_table @@ -45,7 +43,7 @@ Table: list_bucketing_table #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 309 rawDataSize 1482 totalSize 1791 @@ -59,24 +57,19 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_table/part=1/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: CREATE TABLE list_bucketing_table2 (key STRING, value STRING) PARTITIONED BY (part STRING) SKEWED BY (key) ON ("abc") -STORED AS DIRECTORIES PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_table2 POSTHOOK: query: CREATE TABLE list_bucketing_table2 (key STRING, value STRING) PARTITIONED BY (part STRING) SKEWED BY (key) ON ("abc") -STORED AS DIRECTORIES POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_table2 @@ -127,7 +120,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[abc]] Storage Desc Params: diff --git ql/src/test/results/clientpositive/lb_fs_stats.q.out ql/src/test/results/clientpositive/lb_fs_stats.q.out index 65953640aa..8d7f8c49e4 100644 --- ql/src/test/results/clientpositive/lb_fs_stats.q.out +++ ql/src/test/results/clientpositive/lb_fs_stats.q.out @@ -6,11 +6,11 @@ POSTHOOK: query: CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@test_tab -PREHOOK: query: ALTER TABLE test_tab SKEWED BY (key) ON ("484") STORED AS DIRECTORIES +PREHOOK: query: ALTER TABLE test_tab SKEWED BY (key) ON ("484") PREHOOK: type: ALTERTABLE_SKEWED PREHOOK: Input: default@test_tab PREHOOK: Output: default@test_tab -POSTHOOK: query: ALTER TABLE test_tab SKEWED BY (key) ON ("484") STORED AS DIRECTORIES +POSTHOOK: query: ALTER TABLE test_tab SKEWED BY (key) ON ("484") POSTHOOK: type: ALTERTABLE_SKEWED POSTHOOK: Input: default@test_tab POSTHOOK: Output: default@test_tab @@ -45,10 +45,10 @@ Table: test_tab #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5370 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -59,10 +59,7 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/test_tab/part=1/key=484} Storage Desc Params: serialization.format 1 diff --git ql/src/test/results/clientpositive/list_bucket_dml_1.q.out ql/src/test/results/clientpositive/list_bucket_dml_1.q.out index e2f18a165d..af5ba2a479 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_1.q.out @@ -1,14 +1,12 @@ PREHOOK: query: create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") -stored as DIRECTORIES PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_dynamic_part POSTHOOK: query: create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") -stored as DIRECTORIES POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_dynamic_part @@ -302,7 +300,7 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -316,11 +314,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='12') @@ -345,7 +340,7 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -359,11 +354,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=12/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds='2008-04-08' @@ -424,7 +416,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part - numFiles 2 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_dml_11.q.out ql/src/test/results/clientpositive/list_bucket_dml_11.q.out index d708fe8a73..16357ebc43 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_11.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_11.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (value) on ('val_466','val_287','val_82') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (value) on ('val_466','val_287','val_82') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -257,10 +255,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5522 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -271,11 +269,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [value] Skewed Values: [[val_287], [val_466], [val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[val_287]=/list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_287, [val_466]=/list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_466, [val_82]=/list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -307,7 +302,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part - numFiles 4 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -315,7 +310,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 5522 + totalSize 5293 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_12.q.out ql/src/test/results/clientpositive/list_bucket_dml_12.q.out index 83d27c152e..a14dcd04ad 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_12.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_12.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_mul_col POSTHOOK: query: create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -263,10 +261,10 @@ Table: list_bucketing_mul_col #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\",\"col4\":\"true\",\"col5\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 6312 - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### # Storage Information @@ -277,11 +275,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [col2, col4] Skewed Values: [[287, val_287], [466, val_466], [82, val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[287, val_287]=/list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=287/col4=val_287, [466, val_466]=/list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=466/col4=val_466, [82, val_82]=/list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=82/col4=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -315,7 +310,7 @@ STAGE PLANS: columns.types string:string:string:string:string #### A masked pattern was here #### name default.list_bucketing_mul_col - numFiles 4 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -323,7 +318,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -408,7 +403,7 @@ STAGE PLANS: columns.types string:string:string:string:string #### A masked pattern was here #### name default.list_bucketing_mul_col - numFiles 4 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -416,7 +411,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_13.q.out ql/src/test/results/clientpositive/list_bucket_dml_13.q.out index 4d1224bad6..cf9cebe24c 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_13.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_13.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_mul_col POSTHOOK: query: create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string) partitioned by (ds String, hr String) skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -263,10 +261,10 @@ Table: list_bucketing_mul_col #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\",\"col4\":\"true\",\"col5\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 6312 - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### # Storage Information @@ -277,11 +275,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [col2, col4] Skewed Values: [[287, val_287], [466, val_466], [82, val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[287, val_287]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=287/col4=val_287, [466, val_466]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=466/col4=val_466, [82, val_82]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=82/col4=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -315,7 +310,7 @@ STAGE PLANS: columns.types string:string:string:string:string #### A masked pattern was here #### name default.list_bucketing_mul_col - numFiles 4 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -323,7 +318,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 7094 + totalSize 6826 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_14.q.out ql/src/test/results/clientpositive/list_bucket_dml_14.q.out index ed23b0d8b3..a37e58d2aa 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_14.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_14.q.out @@ -1,12 +1,10 @@ PREHOOK: query: create table list_bucketing (key String, value String) skewed by (key) on ("484") -stored as DIRECTORIES PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing POSTHOOK: query: create table list_bucketing (key String, value String) skewed by (key) on ("484") -stored as DIRECTORIES POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing @@ -233,7 +231,7 @@ Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} bucketing_version 2 - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -247,11 +245,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from src @@ -348,7 +343,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 serialization.ddl struct list_bucketing { string key, string value} @@ -370,7 +365,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 serialization.ddl struct list_bucketing { string key, string value} diff --git ql/src/test/results/clientpositive/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/list_bucket_dml_2.q.out index fd8de07cc4..0e8f2e422e 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_2.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -312,10 +310,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -326,11 +324,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -380,7 +375,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part - numFiles 6 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string @@ -388,7 +383,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_3.q.out ql/src/test/results/clientpositive/list_bucket_dml_3.q.out index c75d161012..070daaa841 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_3.q.out @@ -1,8 +1,8 @@ -PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") stored as DIRECTORIES +PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_static_part -POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") stored as DIRECTORIES +POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ("484") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_static_part @@ -293,7 +293,7 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 1000 rawDataSize 10624 totalSize 11624 @@ -307,11 +307,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds='2008-04-08' @@ -372,7 +369,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part - numFiles 4 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_dml_4.q.out ql/src/test/results/clientpositive/list_bucket_dml_4.q.out index 967b84dc42..30e376af53 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_4.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_4.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -312,10 +310,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -326,11 +324,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -751,10 +746,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10786 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -765,11 +760,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -819,7 +811,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part - numFiles 4 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string @@ -827,7 +819,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10786 + totalSize 10586 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_5.q.out ql/src/test/results/clientpositive/list_bucket_dml_5.q.out index d67ec0afeb..888abe70ab 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_5.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_5.q.out @@ -1,14 +1,12 @@ PREHOOK: query: create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@list_bucketing_dynamic_part POSTHOOK: query: create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@list_bucketing_dynamic_part @@ -302,7 +300,7 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -316,11 +314,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='12') @@ -345,7 +340,7 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -359,11 +354,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=12/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=12/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds='2008-04-08' @@ -429,7 +421,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -476,7 +468,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_dml_6.q.out ql/src/test/results/clientpositive/list_bucket_dml_6.q.out index 42afe47d88..3ebf9d7ae7 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_6.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_6.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_dynamic_part POSTHOOK: query: create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -330,7 +328,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] Storage Desc Params: @@ -357,10 +354,10 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10734 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -371,11 +368,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -820,7 +814,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] Storage Desc Params: @@ -847,10 +840,10 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10622 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -861,11 +854,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -962,7 +952,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part - numFiles 4 + numFiles 2 numRows 984 partition_columns ds/hr partition_columns.types string:string @@ -970,7 +960,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_dynamic_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10622 + totalSize 10422 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_7.q.out ql/src/test/results/clientpositive/list_bucket_dml_7.q.out index a325337ac4..2187b9bd9f 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_7.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_7.q.out @@ -1,7 +1,6 @@ PREHOOK: query: CREATE TABLE list_bucketing_dynamic_part (key String, value STRING) PARTITIONED BY (ds string, hr string) skewed by (key) on ('484') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_dynamic_part POSTHOOK: query: CREATE TABLE list_bucketing_dynamic_part (key String, value STRING) PARTITIONED BY (ds string, hr string) skewed by (key) on ('484') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -330,7 +328,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] Storage Desc Params: @@ -357,10 +354,10 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10576 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -371,11 +368,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -820,7 +814,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] Storage Desc Params: @@ -847,10 +840,10 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10520 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -861,11 +854,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -962,7 +952,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part - numFiles 3 + numFiles 2 numRows 984 partition_columns ds/hr partition_columns.types string:string @@ -970,7 +960,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_dynamic_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10520 + totalSize 10422 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_8.q.out ql/src/test/results/clientpositive/list_bucket_dml_8.q.out index 6b407ad476..8bdbe262a4 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_8.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_8.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_dynamic_part POSTHOOK: query: create table list_bucketing_dynamic_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -330,7 +328,6 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] Storage Desc Params: @@ -357,10 +354,10 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 984 rawDataSize 9488 - totalSize 10734 + totalSize 10422 #### A masked pattern was here #### # Storage Information @@ -371,11 +368,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: alter table list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1') concatenate @@ -407,10 +401,10 @@ Database: default Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: - numFiles 3 + numFiles 1 numRows 984 rawDataSize 9488 - totalSize 10586 + totalSize 10386 #### A masked pattern was here #### # Storage Information @@ -421,11 +415,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -525,7 +516,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_dynamic_part - numFiles 3 + numFiles 1 numRows 984 partition_columns ds/hr partition_columns.types string:string @@ -533,7 +524,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_dynamic_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10586 + totalSize 10386 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_dml_9.q.out ql/src/test/results/clientpositive/list_bucket_dml_9.q.out index 7afece8807..e18aeec74c 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_9.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_9.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','103') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','103') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -312,10 +310,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 6 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -326,11 +324,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[103], [484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -751,10 +746,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10786 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -765,11 +760,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[103], [484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -819,7 +811,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part - numFiles 4 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string @@ -827,7 +819,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10786 + totalSize 10586 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out index 40455b771e..4b9715189b 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out @@ -1,14 +1,12 @@ PREHOOK: query: create table fact_daily (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('238','val_238')) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@fact_daily POSTHOOK: query: create table fact_daily (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('238','val_238')) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@fact_daily @@ -46,7 +44,7 @@ Table: fact_daily #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -60,11 +58,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[238, val_238], [484, val_484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[238, val_238]=/fact_daily/ds=1/hr=4/key=238/value=val_238, [484, val_484]=/fact_daily/ds=1/hr=4/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: SELECT count(1) FROM fact_daily WHERE ds='1' and hr='4' @@ -103,7 +98,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -188,7 +183,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -274,7 +269,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -358,7 +353,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out index 54e9009a9f..c1a24f3b00 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out @@ -1,14 +1,12 @@ PREHOOK: query: create table fact_daily (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('238','val_238')) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@fact_daily POSTHOOK: query: create table fact_daily (key String, value String) partitioned by (ds String, hr String) -skewed by (key, value) on (('484','val_484'),('238','val_238')) -stored as DIRECTORIES +skewed by (key, value) on (('484','val_484'),('238','val_238')) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@fact_daily @@ -46,7 +44,7 @@ Table: fact_daily #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -60,11 +58,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[238, val_238], [484, val_484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[238, val_238]=/fact_daily/ds=1/hr=4/key=238/value=val_238, [484, val_484]=/fact_daily/ds=1/hr=4/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: SELECT count(1) FROM fact_daily WHERE ds='1' and hr='4' @@ -103,7 +98,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -188,7 +183,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -276,7 +271,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out index cdf8339bd4..71bb0383d2 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out @@ -58,11 +58,11 @@ Bucket Columns: [] Sort Columns: [] Storage Desc Params: serialization.format 1 -PREHOOK: query: alter table fact_daily skewed by (key, value) on (('484','val_484'),('238','val_238')) stored as DIRECTORIES +PREHOOK: query: alter table fact_daily skewed by (key, value) on (('484','val_484'),('238','val_238')) PREHOOK: type: ALTERTABLE_SKEWED PREHOOK: Input: default@fact_daily PREHOOK: Output: default@fact_daily -POSTHOOK: query: alter table fact_daily skewed by (key, value) on (('484','val_484'),('238','val_238')) stored as DIRECTORIES +POSTHOOK: query: alter table fact_daily skewed by (key, value) on (('484','val_484'),('238','val_238')) POSTHOOK: type: ALTERTABLE_SKEWED POSTHOOK: Input: default@fact_daily POSTHOOK: Output: default@fact_daily @@ -100,7 +100,7 @@ Table: fact_daily #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 3 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -114,18 +114,15 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[238, val_238], [484, val_484]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[238, val_238]=/fact_daily/ds=1/hr=2/key=238/value=val_238, [484, val_484]=/fact_daily/ds=1/hr=2/key=484/value=val_484} Storage Desc Params: serialization.format 1 -PREHOOK: query: alter table fact_daily skewed by (key, value) on (('327','val_327')) stored as DIRECTORIES +PREHOOK: query: alter table fact_daily skewed by (key, value) on (('327','val_327')) PREHOOK: type: ALTERTABLE_SKEWED PREHOOK: Input: default@fact_daily PREHOOK: Output: default@fact_daily -POSTHOOK: query: alter table fact_daily skewed by (key, value) on (('327','val_327')) stored as DIRECTORIES +POSTHOOK: query: alter table fact_daily skewed by (key, value) on (('327','val_327')) POSTHOOK: type: ALTERTABLE_SKEWED POSTHOOK: Input: default@fact_daily POSTHOOK: Output: default@fact_daily @@ -163,7 +160,7 @@ Table: fact_daily #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 500 rawDataSize 5312 totalSize 5812 @@ -177,11 +174,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[327, val_327]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[327, val_327]=/fact_daily/ds=1/hr=3/key=327/value=val_327} Storage Desc Params: serialization.format 1 PREHOOK: query: explain extended @@ -325,7 +319,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 3 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string @@ -412,7 +406,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.fact_daily - numFiles 2 + numFiles 1 numRows 500 partition_columns ds/hr partition_columns.types string:string diff --git ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out index 5e9d191885..064ccde2d8 100644 --- ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -49,11 +47,11 @@ STAGE PLANS: File Output Operator compressed: false GlobalTableId: 1 -#### A masked pattern was here #### + directory: hdfs://### HDFS PATH ### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### + Stats Publishing Key Prefix: hdfs://### HDFS PATH ### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -65,6 +63,7 @@ STAGE PLANS: columns.comments columns.types string:string #### A masked pattern was here #### + location hdfs://### HDFS PATH ### name default.list_bucketing_static_part partition_columns ds/hr partition_columns.types string:string @@ -99,9 +98,9 @@ STAGE PLANS: Execution mode: llap LLAP IO: no inputs Path -> Alias: -#### A masked pattern was here #### + hdfs://### HDFS PATH ### [src] Path -> Partition: -#### A masked pattern was here #### + hdfs://### HDFS PATH ### Partition base file name: src input format: org.apache.hadoop.mapred.TextInputFormat @@ -115,6 +114,7 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### + location hdfs://### HDFS PATH ### name default.src numFiles 1 numRows 500 @@ -137,6 +137,7 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### + location hdfs://### HDFS PATH ### name default.src numFiles 1 numRows 500 @@ -168,10 +169,10 @@ STAGE PLANS: File Output Operator compressed: false GlobalTableId: 0 -#### A masked pattern was here #### + directory: hdfs://### HDFS PATH ### NumFilesPerFileSink: 1 Statistics: Num rows: 1 Data size: 1060 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### + Stats Publishing Key Prefix: hdfs://### HDFS PATH ### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -198,7 +199,7 @@ STAGE PLANS: ds 2008-04-08 hr 11 replace: true -#### A masked pattern was here #### + source: hdfs://### HDFS PATH ### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -210,6 +211,7 @@ STAGE PLANS: columns.comments columns.types string:string #### A masked pattern was here #### + location hdfs://### HDFS PATH ### name default.list_bucketing_static_part partition_columns ds/hr partition_columns.types string:string @@ -223,7 +225,7 @@ STAGE PLANS: Stage: Stage-3 Stats Work Basic Stats Work: -#### A masked pattern was here #### + Stats Aggregation Key Prefix: hdfs://### HDFS PATH ### Column Stats Desc: Columns: key, value Column Types: string, string @@ -271,10 +273,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5520 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -285,10 +287,7 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[103], [484], [51]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484, [51]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=51} Storage Desc Params: serialization.format 1 diff --git ql/src/test/results/clientpositive/llap/mm_all.q.out ql/src/test/results/clientpositive/llap/mm_all.q.out index 8a5c9d0301..0c46c6aae1 100644 --- ql/src/test/results/clientpositive/llap/mm_all.q.out +++ ql/src/test/results/clientpositive/llap/mm_all.q.out @@ -778,12 +778,12 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@partunion_mm POSTHOOK: Output: default@partunion_mm PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +tblproperties ("transactional"="true", "transactional_properties"="insert_only") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@skew_mm POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +tblproperties ("transactional"="true", "transactional_properties"="insert_only") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@skew_mm @@ -829,12 +829,12 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@skew_mm POSTHOOK: Output: default@skew_mm PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@skew_dp_union_mm POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@skew_dp_union_mm diff --git ql/src/test/results/clientpositive/mm_all.q.out ql/src/test/results/clientpositive/mm_all.q.out index f1f4b799a7..f3299b81bb 100644 --- ql/src/test/results/clientpositive/mm_all.q.out +++ ql/src/test/results/clientpositive/mm_all.q.out @@ -792,12 +792,12 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@partunion_mm POSTHOOK: Output: default@partunion_mm PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +tblproperties ("transactional"="true", "transactional_properties"="insert_only") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@skew_mm POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) - stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +tblproperties ("transactional"="true", "transactional_properties"="insert_only") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@skew_mm @@ -843,12 +843,12 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@skew_mm POSTHOOK: Output: default@skew_mm PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only") PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@skew_dp_union_mm POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) -skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only") +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) tblproperties ("transactional"="true", "transactional_properties"="insert_only") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@skew_dp_union_mm diff --git ql/src/test/results/clientpositive/show_create_table_db_table.q.out ql/src/test/results/clientpositive/show_create_table_db_table.q.out index 33410c8582..4bda1d3098 100644 --- ql/src/test/results/clientpositive/show_create_table_db_table.q.out +++ ql/src/test/results/clientpositive/show_create_table_db_table.q.out @@ -26,11 +26,11 @@ POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt2(key string, value int) skewe POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:tmp_feng POSTHOOK: Output: tmp_feng@tmp_showcrt2 -PREHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') stored as directories +PREHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') PREHOOK: type: CREATETABLE PREHOOK: Output: database:tmp_feng PREHOOK: Output: tmp_feng@tmp_showcrt3 -POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') stored as directories +POSTHOOK: query: CREATE TABLE tmp_feng.tmp_showcrt3(key string, value int) skewed by (key) on ('1','2') POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:tmp_feng POSTHOOK: Output: tmp_feng@tmp_showcrt3 @@ -93,7 +93,6 @@ CREATE TABLE `tmp_feng.tmp_showcrt3`( `value` int) SKEWED BY (key) ON (('1'),('2')) - STORED AS DIRECTORIES ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' STORED AS INPUTFORMAT diff --git ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out index e14ae92d1c..89001c9292 100644 --- ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key) on ('484','51','103') - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -45,11 +43,11 @@ STAGE PLANS: File Output Operator compressed: false GlobalTableId: 1 - directory: hdfs://### HDFS PATH ### +#### A masked pattern was here #### NumFilesPerFileSink: 1 Static Partition Specification: ds=2008-04-08/hr=11/ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Stats Publishing Key Prefix: hdfs://### HDFS PATH ### +#### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -61,7 +59,6 @@ STAGE PLANS: columns.comments columns.types string:string #### A masked pattern was here #### - location hdfs://### HDFS PATH ### name default.list_bucketing_static_part partition_columns ds/hr partition_columns.types string:string @@ -76,9 +73,9 @@ STAGE PLANS: MultiFileSpray: false Execution mode: vectorized Path -> Alias: - hdfs://### HDFS PATH ### [src] +#### A masked pattern was here #### Path -> Partition: - hdfs://### HDFS PATH ### +#### A masked pattern was here #### Partition base file name: src input format: org.apache.hadoop.mapred.TextInputFormat @@ -92,7 +89,6 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - location hdfs://### HDFS PATH ### name default.src numFiles 1 numRows 500 @@ -115,7 +111,6 @@ STAGE PLANS: columns.comments 'default','default' columns.types string:string #### A masked pattern was here #### - location hdfs://### HDFS PATH ### name default.src numFiles 1 numRows 500 @@ -138,7 +133,7 @@ STAGE PLANS: ds 2008-04-08 hr 11 replace: true - source: hdfs://### HDFS PATH ### +#### A masked pattern was here #### table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat @@ -150,7 +145,6 @@ STAGE PLANS: columns.comments columns.types string:string #### A masked pattern was here #### - location hdfs://### HDFS PATH ### name default.list_bucketing_static_part partition_columns ds/hr partition_columns.types string:string @@ -164,7 +158,7 @@ STAGE PLANS: Stage: Stage-2 Stats Work Basic Stats Work: - Stats Aggregation Key Prefix: hdfs://### HDFS PATH ### +#### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') select key, value from src @@ -207,10 +201,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5520 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -221,10 +215,7 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key] Skewed Values: [[103], [484], [51]] -Skewed Value to Path: {[103]=hdfs://### HDFS PATH ###, [484]=hdfs://### HDFS PATH ###, [51]=hdfs://### HDFS PATH ###} -Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484, [51]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=51} Storage Desc Params: serialization.format 1 diff --git ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out index 0ee050f9be..67ac5078e7 100644 --- ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out @@ -1,7 +1,6 @@ PREHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -9,7 +8,6 @@ PREHOOK: Output: default@list_bucketing_static_part POSTHOOK: query: create table list_bucketing_static_part (key String, value String) partitioned by (ds String, hr String) skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103')) - stored as DIRECTORIES STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -258,10 +256,10 @@ Table: list_bucketing_static_part #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 6 + numFiles 2 numRows 1000 rawDataSize 9624 - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### # Storage Information @@ -272,11 +270,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [key, value] Skewed Values: [[103, val_103], [484, val_484], [51, val_14]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484} Storage Desc Params: serialization.format 1 PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' @@ -326,7 +321,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.list_bucketing_static_part - numFiles 6 + numFiles 2 numRows 1000 partition_columns ds/hr partition_columns.types string:string @@ -334,7 +329,7 @@ STAGE PLANS: serialization.ddl struct list_bucketing_static_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 10898 + totalSize 10586 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git ql/src/test/results/clientpositive/stats_list_bucket.q.out ql/src/test/results/clientpositive/stats_list_bucket.q.out index 566af31192..946859bb68 100644 --- ql/src/test/results/clientpositive/stats_list_bucket.q.out +++ ql/src/test/results/clientpositive/stats_list_bucket.q.out @@ -11,7 +11,6 @@ PREHOOK: query: create table stats_list_bucket ( c2 string ) partitioned by (ds string, hr string) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -21,7 +20,6 @@ POSTHOOK: query: create table stats_list_bucket ( c2 string ) partitioned by (ds string, hr string) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -60,10 +58,10 @@ Table: stats_list_bucket #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\"}} - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5522 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -74,11 +72,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [c1, c2] Skewed Values: [[287, val_287], [466, val_466], [82, val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[287, val_287]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=287/c2=val_287, [466, val_466]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=466/c2=val_466, [82, val_82]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=82/c2=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: create table stats_list_bucket_1 ( @@ -86,7 +81,6 @@ PREHOOK: query: create table stats_list_bucket_1 ( c2 string ) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -96,7 +90,6 @@ POSTHOOK: query: create table stats_list_bucket_1 ( c2 string ) skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82')) -stored as directories stored as rcfile POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -132,10 +125,10 @@ Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\"}} bucketing_version 2 - numFiles 4 + numFiles 1 numRows 500 rawDataSize 4812 - totalSize 5522 + totalSize 5293 #### A masked pattern was here #### # Storage Information @@ -146,11 +139,8 @@ Compressed: No Num Buckets: -1 Bucket Columns: [] Sort Columns: [] -Stored As SubDirectories: Yes Skewed Columns: [c1, c2] Skewed Values: [[287, val_287], [466, val_466], [82, val_82]] -#### A masked pattern was here #### -Skewed Value to Truncated Path: {[287, val_287]=/stats_list_bucket_1/c1=287/c2=val_287, [466, val_466]=/stats_list_bucket_1/c1=466/c2=val_466, [82, val_82]=/stats_list_bucket_1/c1=82/c2=val_82} Storage Desc Params: serialization.format 1 PREHOOK: query: drop table stats_list_bucket diff --git ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out index c9807b3235..256f149c0e 100644 --- ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out +++ ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out @@ -8,13 +8,11 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@test_tab PREHOOK: query: ALTER TABLE test_tab SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES PREHOOK: type: ALTERTABLE_SKEWED PREHOOK: Input: default@test_tab PREHOOK: Output: default@test_tab POSTHOOK: query: ALTER TABLE test_tab SKEWED BY (key) ON ("484") -STORED AS DIRECTORIES POSTHOOK: type: ALTERTABLE_SKEWED POSTHOOK: Input: default@test_tab POSTHOOK: Output: default@test_tab @@ -74,7 +72,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.test_tab - numFiles 2 + numFiles 1 numRows 500 partition_columns part partition_columns.types string @@ -82,7 +80,7 @@ STAGE PLANS: serialization.ddl struct test_tab { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 1761 + totalSize 1685 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -157,7 +155,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.test_tab - numFiles 2 + numFiles 1 numRows 500 partition_columns part partition_columns.types string @@ -165,7 +163,7 @@ STAGE PLANS: serialization.ddl struct test_tab { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 1761 + totalSize 1685 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe diff --git standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp index af975fce1b..0137f61288 100644 --- standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp +++ standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp @@ -5882,11 +5882,6 @@ void StorageDescriptor::__set_skewedInfo(const SkewedInfo& val) { __isset.skewedInfo = true; } -void StorageDescriptor::__set_storedAsSubDirectories(const bool val) { - this->storedAsSubDirectories = val; -__isset.storedAsSubDirectories = true; -} - uint32_t StorageDescriptor::read(::apache::thrift::protocol::TProtocol* iprot) { apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); @@ -6047,14 +6042,6 @@ uint32_t StorageDescriptor::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; - case 12: - if (ftype == ::apache::thrift::protocol::T_BOOL) { - xfer += iprot->readBool(this->storedAsSubDirectories); - this->__isset.storedAsSubDirectories = true; - } else { - xfer += iprot->skip(ftype); - } - break; default: xfer += iprot->skip(ftype); break; @@ -6150,11 +6137,6 @@ uint32_t StorageDescriptor::write(::apache::thrift::protocol::TProtocol* oprot) xfer += this->skewedInfo.write(oprot); xfer += oprot->writeFieldEnd(); } - if (this->__isset.storedAsSubDirectories) { - xfer += oprot->writeFieldBegin("storedAsSubDirectories", ::apache::thrift::protocol::T_BOOL, 12); - xfer += oprot->writeBool(this->storedAsSubDirectories); - xfer += oprot->writeFieldEnd(); - } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -6173,7 +6155,6 @@ void swap(StorageDescriptor &a, StorageDescriptor &b) { swap(a.sortCols, b.sortCols); swap(a.parameters, b.parameters); swap(a.skewedInfo, b.skewedInfo); - swap(a.storedAsSubDirectories, b.storedAsSubDirectories); swap(a.__isset, b.__isset); } @@ -6189,7 +6170,6 @@ StorageDescriptor::StorageDescriptor(const StorageDescriptor& other232) { sortCols = other232.sortCols; parameters = other232.parameters; skewedInfo = other232.skewedInfo; - storedAsSubDirectories = other232.storedAsSubDirectories; __isset = other232.__isset; } StorageDescriptor& StorageDescriptor::operator=(const StorageDescriptor& other233) { @@ -6204,7 +6184,6 @@ StorageDescriptor& StorageDescriptor::operator=(const StorageDescriptor& other23 sortCols = other233.sortCols; parameters = other233.parameters; skewedInfo = other233.skewedInfo; - storedAsSubDirectories = other233.storedAsSubDirectories; __isset = other233.__isset; return *this; } @@ -6222,7 +6201,6 @@ void StorageDescriptor::printTo(std::ostream& out) const { out << ", " << "sortCols=" << to_string(sortCols); out << ", " << "parameters=" << to_string(parameters); out << ", " << "skewedInfo="; (__isset.skewedInfo ? (out << to_string(skewedInfo)) : (out << "")); - out << ", " << "storedAsSubDirectories="; (__isset.storedAsSubDirectories ? (out << to_string(storedAsSubDirectories)) : (out << "")); out << ")"; } diff --git standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h index 7b42182d60..adbda4535a 100644 --- standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -2925,7 +2925,7 @@ inline std::ostream& operator<<(std::ostream& out, const SkewedInfo& obj) } typedef struct _StorageDescriptor__isset { - _StorageDescriptor__isset() : cols(false), location(false), inputFormat(false), outputFormat(false), compressed(false), numBuckets(false), serdeInfo(false), bucketCols(false), sortCols(false), parameters(false), skewedInfo(false), storedAsSubDirectories(false) {} + _StorageDescriptor__isset() : cols(false), location(false), inputFormat(false), outputFormat(false), compressed(false), numBuckets(false), serdeInfo(false), bucketCols(false), sortCols(false), parameters(false), skewedInfo(false) {} bool cols :1; bool location :1; bool inputFormat :1; @@ -2937,7 +2937,6 @@ typedef struct _StorageDescriptor__isset { bool sortCols :1; bool parameters :1; bool skewedInfo :1; - bool storedAsSubDirectories :1; } _StorageDescriptor__isset; class StorageDescriptor { @@ -2945,7 +2944,7 @@ class StorageDescriptor { StorageDescriptor(const StorageDescriptor&); StorageDescriptor& operator=(const StorageDescriptor&); - StorageDescriptor() : location(), inputFormat(), outputFormat(), compressed(0), numBuckets(0), storedAsSubDirectories(0) { + StorageDescriptor() : location(), inputFormat(), outputFormat(), compressed(0), numBuckets(0) { } virtual ~StorageDescriptor() throw(); @@ -2960,7 +2959,6 @@ class StorageDescriptor { std::vector sortCols; std::map parameters; SkewedInfo skewedInfo; - bool storedAsSubDirectories; _StorageDescriptor__isset __isset; @@ -2986,8 +2984,6 @@ class StorageDescriptor { void __set_skewedInfo(const SkewedInfo& val); - void __set_storedAsSubDirectories(const bool val); - bool operator == (const StorageDescriptor & rhs) const { if (!(cols == rhs.cols)) @@ -3014,10 +3010,6 @@ class StorageDescriptor { return false; else if (__isset.skewedInfo && !(skewedInfo == rhs.skewedInfo)) return false; - if (__isset.storedAsSubDirectories != rhs.__isset.storedAsSubDirectories) - return false; - else if (__isset.storedAsSubDirectories && !(storedAsSubDirectories == rhs.storedAsSubDirectories)) - return false; return true; } bool operator != (const StorageDescriptor &rhs) const { diff --git standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java index 3cfa765fc1..1c7c505b31 100644 --- standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java +++ standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java @@ -49,7 +49,6 @@ private static final org.apache.thrift.protocol.TField SORT_COLS_FIELD_DESC = new org.apache.thrift.protocol.TField("sortCols", org.apache.thrift.protocol.TType.LIST, (short)9); private static final org.apache.thrift.protocol.TField PARAMETERS_FIELD_DESC = new org.apache.thrift.protocol.TField("parameters", org.apache.thrift.protocol.TType.MAP, (short)10); private static final org.apache.thrift.protocol.TField SKEWED_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("skewedInfo", org.apache.thrift.protocol.TType.STRUCT, (short)11); - private static final org.apache.thrift.protocol.TField STORED_AS_SUB_DIRECTORIES_FIELD_DESC = new org.apache.thrift.protocol.TField("storedAsSubDirectories", org.apache.thrift.protocol.TType.BOOL, (short)12); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -68,7 +67,6 @@ private List sortCols; // required private Map parameters; // required private SkewedInfo skewedInfo; // optional - private boolean storedAsSubDirectories; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -82,8 +80,7 @@ BUCKET_COLS((short)8, "bucketCols"), SORT_COLS((short)9, "sortCols"), PARAMETERS((short)10, "parameters"), - SKEWED_INFO((short)11, "skewedInfo"), - STORED_AS_SUB_DIRECTORIES((short)12, "storedAsSubDirectories"); + SKEWED_INFO((short)11, "skewedInfo"); private static final Map byName = new HashMap(); @@ -120,8 +117,6 @@ public static _Fields findByThriftId(int fieldId) { return PARAMETERS; case 11: // SKEWED_INFO return SKEWED_INFO; - case 12: // STORED_AS_SUB_DIRECTORIES - return STORED_AS_SUB_DIRECTORIES; default: return null; } @@ -164,9 +159,8 @@ public String getFieldName() { // isset id assignments private static final int __COMPRESSED_ISSET_ID = 0; private static final int __NUMBUCKETS_ISSET_ID = 1; - private static final int __STOREDASSUBDIRECTORIES_ISSET_ID = 2; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.SKEWED_INFO,_Fields.STORED_AS_SUB_DIRECTORIES}; + private static final _Fields optionals[] = {_Fields.SKEWED_INFO}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -197,8 +191,6 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); tmpMap.put(_Fields.SKEWED_INFO, new org.apache.thrift.meta_data.FieldMetaData("skewedInfo", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, SkewedInfo.class))); - tmpMap.put(_Fields.STORED_AS_SUB_DIRECTORIES, new org.apache.thrift.meta_data.FieldMetaData("storedAsSubDirectories", org.apache.thrift.TFieldRequirementType.OPTIONAL, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(StorageDescriptor.class, metaDataMap); } @@ -277,7 +269,6 @@ public StorageDescriptor(StorageDescriptor other) { if (other.isSetSkewedInfo()) { this.skewedInfo = new SkewedInfo(other.skewedInfo); } - this.storedAsSubDirectories = other.storedAsSubDirectories; } public StorageDescriptor deepCopy() { @@ -299,8 +290,6 @@ public void clear() { this.sortCols = null; this.parameters = null; this.skewedInfo = null; - setStoredAsSubDirectoriesIsSet(false); - this.storedAsSubDirectories = false; } public int getColsSize() { @@ -610,28 +599,6 @@ public void setSkewedInfoIsSet(boolean value) { } } - public boolean isStoredAsSubDirectories() { - return this.storedAsSubDirectories; - } - - public void setStoredAsSubDirectories(boolean storedAsSubDirectories) { - this.storedAsSubDirectories = storedAsSubDirectories; - setStoredAsSubDirectoriesIsSet(true); - } - - public void unsetStoredAsSubDirectories() { - __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __STOREDASSUBDIRECTORIES_ISSET_ID); - } - - /** Returns true if field storedAsSubDirectories is set (has been assigned a value) and false otherwise */ - public boolean isSetStoredAsSubDirectories() { - return EncodingUtils.testBit(__isset_bitfield, __STOREDASSUBDIRECTORIES_ISSET_ID); - } - - public void setStoredAsSubDirectoriesIsSet(boolean value) { - __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __STOREDASSUBDIRECTORIES_ISSET_ID, value); - } - public void setFieldValue(_Fields field, Object value) { switch (field) { case COLS: @@ -722,14 +689,6 @@ public void setFieldValue(_Fields field, Object value) { } break; - case STORED_AS_SUB_DIRECTORIES: - if (value == null) { - unsetStoredAsSubDirectories(); - } else { - setStoredAsSubDirectories((Boolean)value); - } - break; - } } @@ -768,9 +727,6 @@ public Object getFieldValue(_Fields field) { case SKEWED_INFO: return getSkewedInfo(); - case STORED_AS_SUB_DIRECTORIES: - return isStoredAsSubDirectories(); - } throw new IllegalStateException(); } @@ -804,8 +760,6 @@ public boolean isSet(_Fields field) { return isSetParameters(); case SKEWED_INFO: return isSetSkewedInfo(); - case STORED_AS_SUB_DIRECTORIES: - return isSetStoredAsSubDirectories(); } throw new IllegalStateException(); } @@ -922,15 +876,6 @@ public boolean equals(StorageDescriptor that) { return false; } - boolean this_present_storedAsSubDirectories = true && this.isSetStoredAsSubDirectories(); - boolean that_present_storedAsSubDirectories = true && that.isSetStoredAsSubDirectories(); - if (this_present_storedAsSubDirectories || that_present_storedAsSubDirectories) { - if (!(this_present_storedAsSubDirectories && that_present_storedAsSubDirectories)) - return false; - if (this.storedAsSubDirectories != that.storedAsSubDirectories) - return false; - } - return true; } @@ -993,11 +938,6 @@ public int hashCode() { if (present_skewedInfo) list.add(skewedInfo); - boolean present_storedAsSubDirectories = true && (isSetStoredAsSubDirectories()); - list.add(present_storedAsSubDirectories); - if (present_storedAsSubDirectories) - list.add(storedAsSubDirectories); - return list.hashCode(); } @@ -1119,16 +1059,6 @@ public int compareTo(StorageDescriptor other) { return lastComparison; } } - lastComparison = Boolean.valueOf(isSetStoredAsSubDirectories()).compareTo(other.isSetStoredAsSubDirectories()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetStoredAsSubDirectories()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.storedAsSubDirectories, other.storedAsSubDirectories); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -1230,12 +1160,6 @@ public String toString() { } first = false; } - if (isSetStoredAsSubDirectories()) { - if (!first) sb.append(", "); - sb.append("storedAsSubDirectories:"); - sb.append(this.storedAsSubDirectories); - first = false; - } sb.append(")"); return sb.toString(); } @@ -1421,14 +1345,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, StorageDescriptor s org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 12: // STORED_AS_SUB_DIRECTORIES - if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { - struct.storedAsSubDirectories = iprot.readBool(); - struct.setStoredAsSubDirectoriesIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1524,11 +1440,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, StorageDescriptor oprot.writeFieldEnd(); } } - if (struct.isSetStoredAsSubDirectories()) { - oprot.writeFieldBegin(STORED_AS_SUB_DIRECTORIES_FIELD_DESC); - oprot.writeBool(struct.storedAsSubDirectories); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1580,10 +1491,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor s if (struct.isSetSkewedInfo()) { optionals.set(10); } - if (struct.isSetStoredAsSubDirectories()) { - optionals.set(11); - } - oprot.writeBitSet(optionals, 12); + oprot.writeBitSet(optionals, 11); if (struct.isSetCols()) { { oprot.writeI32(struct.cols.size()); @@ -1642,15 +1550,12 @@ public void write(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor s if (struct.isSetSkewedInfo()) { struct.skewedInfo.write(oprot); } - if (struct.isSetStoredAsSubDirectories()) { - oprot.writeBool(struct.storedAsSubDirectories); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(12); + BitSet incoming = iprot.readBitSet(11); if (incoming.get(0)) { { org.apache.thrift.protocol.TList _list185 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); @@ -1737,10 +1642,6 @@ public void read(org.apache.thrift.protocol.TProtocol prot, StorageDescriptor st struct.skewedInfo.read(iprot); struct.setSkewedInfoIsSet(true); } - if (incoming.get(11)) { - struct.storedAsSubDirectories = iprot.readBool(); - struct.setStoredAsSubDirectoriesIsSet(true); - } } } diff --git standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php index 353c0deb91..4f28db2549 100644 --- standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php +++ standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -5909,10 +5909,6 @@ class StorageDescriptor { * @var \metastore\SkewedInfo */ public $skewedInfo = null; - /** - * @var bool - */ - public $storedAsSubDirectories = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -5985,10 +5981,6 @@ class StorageDescriptor { 'type' => TType::STRUCT, 'class' => '\metastore\SkewedInfo', ), - 12 => array( - 'var' => 'storedAsSubDirectories', - 'type' => TType::BOOL, - ), ); } if (is_array($vals)) { @@ -6025,9 +6017,6 @@ class StorageDescriptor { if (isset($vals['skewedInfo'])) { $this->skewedInfo = $vals['skewedInfo']; } - if (isset($vals['storedAsSubDirectories'])) { - $this->storedAsSubDirectories = $vals['storedAsSubDirectories']; - } } } @@ -6174,13 +6163,6 @@ class StorageDescriptor { $xfer += $input->skip($ftype); } break; - case 12: - if ($ftype == TType::BOOL) { - $xfer += $input->readBool($this->storedAsSubDirectories); - } else { - $xfer += $input->skip($ftype); - } - break; default: $xfer += $input->skip($ftype); break; @@ -6304,11 +6286,6 @@ class StorageDescriptor { $xfer += $this->skewedInfo->write($output); $xfer += $output->writeFieldEnd(); } - if ($this->storedAsSubDirectories !== null) { - $xfer += $output->writeFieldBegin('storedAsSubDirectories', TType::BOOL, 12); - $xfer += $output->writeBool($this->storedAsSubDirectories); - $xfer += $output->writeFieldEnd(); - } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py index fdec32e10c..6d5af61278 100644 --- standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -4205,7 +4205,6 @@ class StorageDescriptor: - sortCols - parameters - skewedInfo - - storedAsSubDirectories """ thrift_spec = ( @@ -4221,10 +4220,9 @@ class StorageDescriptor: (9, TType.LIST, 'sortCols', (TType.STRUCT,(Order, Order.thrift_spec)), None, ), # 9 (10, TType.MAP, 'parameters', (TType.STRING,None,TType.STRING,None), None, ), # 10 (11, TType.STRUCT, 'skewedInfo', (SkewedInfo, SkewedInfo.thrift_spec), None, ), # 11 - (12, TType.BOOL, 'storedAsSubDirectories', None, None, ), # 12 ) - def __init__(self, cols=None, location=None, inputFormat=None, outputFormat=None, compressed=None, numBuckets=None, serdeInfo=None, bucketCols=None, sortCols=None, parameters=None, skewedInfo=None, storedAsSubDirectories=None,): + def __init__(self, cols=None, location=None, inputFormat=None, outputFormat=None, compressed=None, numBuckets=None, serdeInfo=None, bucketCols=None, sortCols=None, parameters=None, skewedInfo=None,): self.cols = cols self.location = location self.inputFormat = inputFormat @@ -4236,7 +4234,6 @@ def __init__(self, cols=None, location=None, inputFormat=None, outputFormat=None self.sortCols = sortCols self.parameters = parameters self.skewedInfo = skewedInfo - self.storedAsSubDirectories = storedAsSubDirectories def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -4327,11 +4324,6 @@ def read(self, iprot): self.skewedInfo.read(iprot) else: iprot.skip(ftype) - elif fid == 12: - if ftype == TType.BOOL: - self.storedAsSubDirectories = iprot.readBool() - else: - iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -4399,10 +4391,6 @@ def write(self, oprot): oprot.writeFieldBegin('skewedInfo', TType.STRUCT, 11) self.skewedInfo.write(oprot) oprot.writeFieldEnd() - if self.storedAsSubDirectories is not None: - oprot.writeFieldBegin('storedAsSubDirectories', TType.BOOL, 12) - oprot.writeBool(self.storedAsSubDirectories) - oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -4423,7 +4411,6 @@ def __hash__(self): value = (value * 31) ^ hash(self.sortCols) value = (value * 31) ^ hash(self.parameters) value = (value * 31) ^ hash(self.skewedInfo) - value = (value * 31) ^ hash(self.storedAsSubDirectories) return value def __repr__(self): diff --git standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb index fb73b28f62..ed25419aab 100644 --- standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -997,7 +997,6 @@ class StorageDescriptor SORTCOLS = 9 PARAMETERS = 10 SKEWEDINFO = 11 - STOREDASSUBDIRECTORIES = 12 FIELDS = { COLS => {:type => ::Thrift::Types::LIST, :name => 'cols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}}, @@ -1010,8 +1009,7 @@ class StorageDescriptor BUCKETCOLS => {:type => ::Thrift::Types::LIST, :name => 'bucketCols', :element => {:type => ::Thrift::Types::STRING}}, SORTCOLS => {:type => ::Thrift::Types::LIST, :name => 'sortCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Order}}, PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}, - SKEWEDINFO => {:type => ::Thrift::Types::STRUCT, :name => 'skewedInfo', :class => ::SkewedInfo, :optional => true}, - STOREDASSUBDIRECTORIES => {:type => ::Thrift::Types::BOOL, :name => 'storedAsSubDirectories', :optional => true} + SKEWEDINFO => {:type => ::Thrift::Types::STRUCT, :name => 'skewedInfo', :class => ::SkewedInfo, :optional => true} } def struct_fields; FIELDS; end diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 56fbfed944..19e5dfaf57 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -628,7 +628,7 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw "select " + PARTITIONS + ".\"PART_ID\", " + SDS + ".\"SD_ID\", " + SDS + ".\"CD_ID\"," + " " + SERDES + ".\"SERDE_ID\", " + PARTITIONS + ".\"CREATE_TIME\"," + " " + PARTITIONS + ".\"LAST_ACCESS_TIME\", " + SDS + ".\"INPUT_FORMAT\", " + SDS + ".\"IS_COMPRESSED\"," - + " " + SDS + ".\"IS_STOREDASSUBDIRECTORIES\", " + SDS + ".\"LOCATION\", " + SDS + ".\"NUM_BUCKETS\"," + + " " + SDS + ".\"LOCATION\", " + SDS + ".\"NUM_BUCKETS\"," + " " + SDS + ".\"OUTPUT_FORMAT\", " + SERDES + ".\"NAME\", " + SERDES + ".\"SLIB\" " + "from " + PARTITIONS + "" + " left outer join " + SDS + " on " + PARTITIONS + ".\"SD_ID\" = " + SDS + ".\"SD_ID\" " @@ -701,11 +701,9 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw sd.setInputFormat((String)fields[6]); Boolean tmpBoolean = extractSqlBoolean(fields[7]); if (tmpBoolean != null) sd.setCompressed(tmpBoolean); - tmpBoolean = extractSqlBoolean(fields[8]); - if (tmpBoolean != null) sd.setStoredAsSubDirectories(tmpBoolean); - sd.setLocation((String)fields[9]); - if (fields[10] != null) sd.setNumBuckets(extractSqlInt(fields[10])); - sd.setOutputFormat((String)fields[11]); + sd.setLocation((String)fields[8]); + if (fields[10] != null) sd.setNumBuckets(extractSqlInt(fields[9])); + sd.setOutputFormat((String)fields[10]); sdSb.append(sdId).append(","); part.setSd(sd); @@ -727,8 +725,8 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw throw new MetaException("SDs reuse serdes; we don't expect that"); } serde.setParameters(new HashMap()); - serde.setName((String)fields[12]); - serde.setSerializationLib((String)fields[13]); + serde.setName((String)fields[11]); + serde.setSerializationLib((String)fields[12]); serdeSb.append(serdeId).append(","); sd.setSerdeInfo(serde); Deadline.checkTimeout(); diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index b0a805f047..14de0cb971 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -2069,7 +2069,6 @@ private StorageDescriptor convertToStorageDescriptor( convertToSkewedValues(msd.getSkewedColValues()), covertToSkewedMap(msd.getSkewedColValueLocationMaps())); sd.setSkewedInfo(skewedInfo); - sd.setStoredAsSubDirectories(msd.isStoredAsSubDirectories()); return sd; } @@ -2180,7 +2179,7 @@ private MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd, convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValues()), covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() - .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories()); + .getSkewedColValueLocationMaps())); } private MCreationMetadata convertToMCreationMetadata( @@ -4130,7 +4129,6 @@ private void copyMSD(MStorageDescriptor newSd, MStorageDescriptor oldSd) { oldSd.setSkewedColValueLocationMaps(newSd.getSkewedColValueLocationMaps()); oldSd.setSortCols(newSd.getSortCols()); oldSd.setParameters(newSd.getParameters()); - oldSd.setStoredAsSubDirectories(newSd.isStoredAsSubDirectories()); } /** diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java index 433e7c7c18..7eb4f0e565 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java @@ -41,7 +41,7 @@ private String location, inputFormat, outputFormat; private int numBuckets; private Map storageDescriptorParams; - private boolean compressed, storedAsSubDirectories; + private boolean compressed; private List bucketCols, skewedColNames; private List sortCols; private List> skewedColValues; @@ -64,7 +64,6 @@ protected StorageDescriptorBuilder() { protected StorageDescriptor buildSd() throws MetaException { StorageDescriptor sd = new StorageDescriptor(getCols(), location, inputFormat, outputFormat, compressed, numBuckets, buildSerde(), bucketCols, sortCols, storageDescriptorParams); - sd.setStoredAsSubDirectories(storedAsSubDirectories); if (skewedColNames != null) { SkewedInfo skewed = new SkewedInfo(skewedColNames, skewedColValues, skewedColValueLocationMaps); @@ -110,11 +109,6 @@ public T setCompressed(boolean compressed) { return child; } - public T setStoredAsSubDirectories(boolean storedAsSubDirectories) { - this.storedAsSubDirectories = storedAsSubDirectories; - return child; - } - public T setBucketCols(List bucketCols) { this.bucketCols = bucketCols; return child; diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MStorageDescriptor.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MStorageDescriptor.java index 4c6ce008f8..91e9dcf856 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MStorageDescriptor.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MStorageDescriptor.java @@ -35,7 +35,6 @@ private List skewedColNames; private List skewedColValues; private Map skewedColValueLocationMaps; - private boolean isStoredAsSubDirectories; public MStorageDescriptor() {} @@ -56,7 +55,7 @@ public MStorageDescriptor(MColumnDescriptor cd, String location, String inputFor String outputFormat, boolean isCompressed, int numBuckets, MSerDeInfo serDeInfo, List bucketCols, List sortOrder, Map parameters, List skewedColNames, List skewedColValues, - Map skewedColValueLocationMaps, boolean storedAsSubDirectories) { + Map skewedColValueLocationMaps) { this.cd = cd; this.location = location; this.inputFormat = inputFormat; @@ -70,7 +69,6 @@ public MStorageDescriptor(MColumnDescriptor cd, String location, String inputFor this.skewedColNames = skewedColNames; this.skewedColValues = skewedColValues; this.skewedColValueLocationMaps = skewedColValueLocationMaps; - this.isStoredAsSubDirectories = storedAsSubDirectories; } @@ -257,21 +255,4 @@ public void setSkewedColValues(List skewedColValues) { public void setSkewedColValueLocationMaps(Map listBucketColValuesMapping) { this.skewedColValueLocationMaps = listBucketColValuesMapping; } - - - /** - * @return the storedAsSubDirectories - */ - public boolean isStoredAsSubDirectories() { - return isStoredAsSubDirectories; - } - - - /** - * @param storedAsSubDirectories the storedAsSubDirectories to set - */ - public void setStoredAsSubDirectories(boolean storedAsSubDirectories) { - this.isStoredAsSubDirectories = storedAsSubDirectories; - } - } diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 742b6bf76b..a9473347a6 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -442,7 +442,7 @@ private static String getPartitionValWithInvalidCharacter(List partVals, md.update(e.getValue().getBytes(ENCODING)); } } - md.update(sd.isStoredAsSubDirectories() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING)); + md.update("false".getBytes(ENCODING)); } return md.digest(); diff --git standalone-metastore/src/main/resources/package.jdo standalone-metastore/src/main/resources/package.jdo index 2d2cb19991..122addf4b7 100644 --- standalone-metastore/src/main/resources/package.jdo +++ standalone-metastore/src/main/resources/package.jdo @@ -381,9 +381,6 @@ - - - diff --git standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql index e818e1b7e2..7278b03666 100644 --- standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql +++ standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql @@ -81,7 +81,7 @@ CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VA CREATE TABLE "APP"."PART_COL_PRIVS" ("PART_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_COL_PRIV" VARCHAR(128)); -CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT, "IS_STOREDASSUBDIRECTORIES" CHAR(1) NOT NULL); +CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT); CREATE TABLE "APP"."SEQUENCE_TABLE" ("SEQUENCE_NAME" VARCHAR(256) NOT NULL, "NEXT_VAL" BIGINT NOT NULL); diff --git standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql index c88fb18973..58deab9ef2 100644 --- standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql +++ standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql @@ -404,7 +404,6 @@ CREATE TABLE SDS CD_ID bigint NULL, INPUT_FORMAT nvarchar(4000) NULL, IS_COMPRESSED bit NOT NULL, - IS_STOREDASSUBDIRECTORIES bit NOT NULL, LOCATION nvarchar(4000) NULL, NUM_BUCKETS int NOT NULL, OUTPUT_FORMAT nvarchar(4000) NULL, diff --git standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index c54df55ed9..137276b8a8 100644 --- standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -396,7 +396,6 @@ CREATE TABLE IF NOT EXISTS `SDS` ( `CD_ID` bigint(20) DEFAULT NULL, `INPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `IS_COMPRESSED` bit(1) NOT NULL, - `IS_STOREDASSUBDIRECTORIES` bit(1) NOT NULL, `LOCATION` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `NUM_BUCKETS` int(11) NOT NULL, `OUTPUT_FORMAT` varchar(4000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, diff --git standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index 63cc1f75e2..04b44e6bf9 100644 --- standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -267,8 +267,7 @@ CREATE TABLE SDS LOCATION VARCHAR2(4000) NULL, NUM_BUCKETS NUMBER (10) NOT NULL, OUTPUT_FORMAT VARCHAR2(4000) NULL, - SERDE_ID NUMBER NULL, - IS_STOREDASSUBDIRECTORIES NUMBER(1) NOT NULL CHECK (IS_STOREDASSUBDIRECTORIES IN (1,0)) + SERDE_ID NUMBER NULL ); ALTER TABLE SDS ADD CONSTRAINT SDS_PK PRIMARY KEY (SD_ID); diff --git standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql index d210a55fcf..11430c9a60 100644 --- standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql +++ standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql @@ -295,8 +295,7 @@ CREATE TABLE "SDS" ( "NUM_BUCKETS" bigint NOT NULL, "OUTPUT_FORMAT" character varying(4000) DEFAULT NULL::character varying, "SERDE_ID" bigint, - "CD_ID" bigint, - "IS_STOREDASSUBDIRECTORIES" boolean NOT NULL + "CD_ID" bigint ); diff --git standalone-metastore/src/main/thrift/hive_metastore.thrift standalone-metastore/src/main/thrift/hive_metastore.thrift index 19d4433078..d78776bd00 100644 --- standalone-metastore/src/main/thrift/hive_metastore.thrift +++ standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -401,8 +401,7 @@ struct StorageDescriptor { 8: list bucketCols, // reducer grouping columns and clustering columns and bucketing columns` 9: list sortCols, // sort order of the data in each bucket 10: map parameters, // any user supplied key value hash - 11: optional SkewedInfo skewedInfo, // skewed information - 12: optional bool storedAsSubDirectories // stored as subdirectories or not + 11: optional SkewedInfo skewedInfo } // table information diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index cb32236d54..d8c2e8cd70 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -1857,7 +1857,6 @@ public void testComplexTable() throws Exception { assertEquals(tbl2.getTableName(), tblName); assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size()); assertFalse(tbl2.getSd().isCompressed()); - assertFalse(tbl2.getSd().isStoredAsSubDirectories()); assertEquals(tbl2.getSd().getNumBuckets(), 1); assertEquals("Use this for comments etc", tbl2.getSd().getParameters() diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index 62ed380dfc..ee088f5f68 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -52,14 +52,11 @@ import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; -import jline.internal.Log; - import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; @Category(MetastoreCheckinTest.class) @@ -1054,7 +1051,6 @@ private Table createTestTbl(String dbName, String tblName, String tblOwner, SerDeInfo serdeInfo = new SerDeInfo("serde", "seriallib", new HashMap<>()); StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serdeInfo, null, null, serdeParams); - sd.setStoredAsSubDirectories(false); Table tbl = new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, ptnCols, tblParams, null, null, TableType.MANAGED_TABLE.toString()); tbl.setCatName(DEFAULT_CATALOG_NAME); diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java index 88064d920f..d835dbe406 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java @@ -1440,7 +1440,6 @@ private Table createTable(String dbName, String tableName, List par .setPartCols(partCols) .addStorageDescriptorParam("partTestSDParamKey", "partTestSDParamValue") .setSerdeName(tableName) - .setStoredAsSubDirectories(false) .addSerdeParam("partTestSerdeParamKey", "partTestSerdeParamValue") .setLocation(location) .create(client, metaStore.getConf()); @@ -1580,8 +1579,6 @@ private void verifyPartitionAttributesDefaultValues(Partition partition, String "org.apache.hadoop.hive.ql.io.HiveOutputFormat", sd.getOutputFormat()); Assert.assertFalse("The compressed attribute doesn't have the default value.", sd.isCompressed()); - Assert.assertFalse("The storedAsSubDirectories attribute doesn't have the default value.", - sd.isStoredAsSubDirectories()); Assert.assertEquals("The numBuckets attribute doesn't have the default value.", 0, sd.getNumBuckets()); Assert.assertTrue("The default value of the attribute 'bucketCols' should be an empty list.", diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java index debcd0eee9..fd87b463f6 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java @@ -916,7 +916,6 @@ private Table createTable(String dbName, String tableName, List par .setPartCols(partCols) .addStorageDescriptorParam("partTestSDParamKey", "partTestSDParamValue") .setSerdeName(tableName) - .setStoredAsSubDirectories(false) .addSerdeParam("partTestSerdeParamKey", "partTestSerdeParamValue") .setLocation(location) .create(client, metaStore.getConf()); diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java index fe2d7587f6..9eb1838113 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java @@ -296,7 +296,6 @@ public void testCreateTableDefaultValues() throws Exception { Assert.assertEquals("Storage descriptor bucket cols", 0, createdSd.getBucketCols().size()); Assert.assertEquals("Storage descriptor sort cols", 0, createdSd.getSortCols().size()); Assert.assertEquals("Storage descriptor parameters", 0, createdSd.getParameters().size()); - Assert.assertFalse("Storage descriptor stored as subdir", createdSd.isStoredAsSubDirectories()); // Serde info SerDeInfo serDeInfo = createdSd.getSerdeInfo(); @@ -1356,7 +1355,6 @@ private Table getTableWithAllParametersSet() throws MetaException { .setViewOriginalText("viewOriginalText") .setSerdeLib("serdelib") .setSerdeName("serdename") - .setStoredAsSubDirectories(true) .addSerdeParam("serdeParam", "serdeParamValue") .addTableParam("tableParam", "tableParamValue") .addStorageDescriptorParam("sdParam", "sdParamValue") diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java index 9e425cff06..75722878e6 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java @@ -341,10 +341,10 @@ public void testValidateLocations() throws Exception { "insert into CTLGS values (1, 'mycat', 'mydescription', 'hdfs://myhost.com:8020/user/hive/warehouse');", "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');", "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'mycat');", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n');", @@ -371,17 +371,17 @@ public void testValidateLocations() throws Exception { "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'mycat');", "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'mycat');", "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'mycat');", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3000 ,1435255431,2,0 ,'hive',0,3000,'mytal3000','MANAGED_TABLE',NULL,NULL,'n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4011 ,1435255431,4,0 ,'hive',0,4001,'mytal4011','MANAGED_TABLE',NULL,NULL,'n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,4,0 ,'hive',0,4002,'','MANAGED_TABLE',NULL,NULL,'n');", @@ -469,8 +469,8 @@ private void createTestHiveTableSchemas() throws IOException { String[] scripts = new String[] { "insert into CTLGS values (1, 'mycat', 'my description', 'hdfs://myhost.com:8020/user/hive/warehouse');", "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", - "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", + "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');", "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,2,'aTable','MANAGED_TABLE',NULL,NULL,'n');", "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);"