diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 1a2b3c1f6c..0940b0c2d2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1877,11 +1877,13 @@ else if(!isAcidIUDoperation && isFullAcidTable) { // base_x. (there is Insert Overwrite and Load Data Overwrite) boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); replaceFiles(tbl.getPath(), loadPath, destPath, oldPartPath, getConf(), - isSrcLocal, isAutoPurge, newFiles, filter, isMmTableWrite?true:false); + isSrcLocal, isAutoPurge, newFiles, filter, isMmTableWrite?true:false, + tbl.getNumBuckets() > 0 ? true : false); } else { FileSystem fs = tbl.getDataLocation().getFileSystem(conf); copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation, - (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles); + (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles, + tbl.getNumBuckets() > 0 ? true : false); } } perfLogger.PerfLogEnd("MoveTask", "FileMoves"); @@ -2422,12 +2424,14 @@ else if(!isAcidIUDoperation && isFullAcidTable) { //todo: should probably do the same for MM IOW boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); replaceFiles(tblPath, loadPath, destPath, tblPath, - sessionConf, isSrcLocal, isAutopurge, newFiles, filter, isMmTable?true:false); + sessionConf, isSrcLocal, isAutopurge, newFiles, filter, isMmTable?true:false, + tbl.getNumBuckets() > 0 ? true : false); } else { try { FileSystem fs = tbl.getDataLocation().getFileSystem(sessionConf); copyFiles(sessionConf, loadPath, destPath, fs, isSrcLocal, isAcidIUDoperation, - loadFileType == LoadFileType.OVERWRITE_EXISTING, newFiles); + loadFileType == LoadFileType.OVERWRITE_EXISTING, newFiles, + tbl.getNumBuckets() > 0 ? true : false); } catch (IOException e) { throw new HiveException("addFiles: filesystem error in check phase", e); } @@ -3283,9 +3287,13 @@ public PrincipalPrivilegeSet get_privilege_set(HiveObjectType objectType, } } + private static String getPathName(int taskId) { + return Utilities.replaceTaskId("000000", taskId) + "_0"; + } + private static void copyFiles(final HiveConf conf, final FileSystem destFs, FileStatus[] srcs, final FileSystem srcFs, final Path destf, final boolean isSrcLocal, - boolean isOverwrite, final List newFiles) throws HiveException { + boolean isOverwrite, final List newFiles, boolean isBucketed) throws HiveException { final HdfsUtils.HadoopFileStatus fullDestStatus; try { @@ -3301,6 +3309,7 @@ private static void copyFiles(final HiveConf conf, final FileSystem destFs, final ExecutorService pool = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25) > 0 ? Executors.newFixedThreadPool(conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25), new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Move-Thread-%d").build()) : null; + int taskId = isBucketed? -1 : 0; for (FileStatus src : srcs) { FileStatus[] files; if (src.isDirectory()) { @@ -3328,7 +3337,8 @@ private static void copyFiles(final HiveConf conf, final FileSystem destFs, // copy from source to destination, we will inherit the destination's parent group ownership. if (null == pool) { try { - Path destPath = mvFile(conf, srcFs, srcP, destFs, destf, isSrcLocal, isOverwrite, isRenameAllowed); + Path destPath = mvFile(conf, srcFs, srcP, destFs, destf, isSrcLocal, isOverwrite, isRenameAllowed, + isBucketed, taskId++); if (null != newFiles) { newFiles.add(destPath); @@ -3337,6 +3347,8 @@ private static void copyFiles(final HiveConf conf, final FileSystem destFs, throw getHiveException(e, msg, "Failed to move: {}"); } } else { + // future only takes final or seemingly final values. Make a final copy of taskId + final int finalTaskId = taskId++; futures.add(pool.submit(new Callable>() { @Override public ObjectPair call() throws HiveException { @@ -3344,7 +3356,7 @@ private static void copyFiles(final HiveConf conf, final FileSystem destFs, try { Path destPath = - mvFile(conf, srcFs, srcP, destFs, destf, isSrcLocal, isOverwrite, isRenameAllowed); + mvFile(conf, srcFs, srcP, destFs, destf, isSrcLocal, isOverwrite, isRenameAllowed, isBucketed, finalTaskId); if (null != newFiles) { newFiles.add(destPath); @@ -3441,12 +3453,20 @@ private static Path getQualifiedPathWithoutSchemeAndAuthority(Path srcf, FileSys * @throws IOException if there was an issue moving the file */ private static Path mvFile(HiveConf conf, FileSystem sourceFs, Path sourcePath, FileSystem destFs, Path destDirPath, - boolean isSrcLocal, boolean isOverwrite, boolean isRenameAllowed) throws IOException { + boolean isSrcLocal, boolean isOverwrite, boolean isRenameAllowed, + boolean isBucketed, int taskId) throws IOException { // Strip off the file type, if any so we don't make: // 000000_0.gz -> 000000_0.gz_copy_1 final String fullname = sourcePath.getName(); - final String name = FilenameUtils.getBaseName(sourcePath.getName()); + //final String name = FilenameUtils.getBaseName(sourcePath.getName()); + final String name; + if (isBucketed) { + name = FilenameUtils.getBaseName(sourcePath.getName()); + } else { + name = getPathName(taskId); + } + final String type = FilenameUtils.getExtension(sourcePath.getName()); Path destFilePath = new Path(destDirPath, fullname); @@ -3759,7 +3779,7 @@ static protected boolean needToCopy(Path srcf, Path destf, FileSystem srcFs, Fil */ static protected void copyFiles(HiveConf conf, Path srcf, Path destf, FileSystem fs, boolean isSrcLocal, boolean isAcid, - boolean isOverwrite, List newFiles) throws HiveException { + boolean isOverwrite, List newFiles, boolean isBucketed) throws HiveException { try { // create the destination if it does not exist if (!fs.exists(destf)) { @@ -3791,7 +3811,7 @@ static protected void copyFiles(HiveConf conf, Path srcf, Path destf, FileSystem if (isAcid) { moveAcidFiles(srcFs, srcs, destf, newFiles); } else { - copyFiles(conf, fs, srcs, srcFs, destf, isSrcLocal, isOverwrite, newFiles); + copyFiles(conf, fs, srcs, srcFs, destf, isSrcLocal, isOverwrite, newFiles, isBucketed); } } @@ -3920,6 +3940,34 @@ private static void moveAcidFiles(String deltaFileType, PathFilter pathFilter, F } } + private static void moveFiles(FileStatus[] files, Path srcf, Path destf, + List newFiles, HiveConf conf, + boolean isSrcLocal, boolean isBucketed) + throws HiveException { + int taskId = 0; + try { + for (FileStatus src : files) { + Path destFile; + if (isBucketed) { + destFile = new Path(destf, src.getPath().getName()); + } else { + destFile = new Path(destf, getPathName(taskId)); + } + if (!moveFile(conf, src.getPath(), destFile, true, isSrcLocal)) { + throw new IOException("Error moving: " + srcf + " into: " + destf); + } + + // Add file paths of the files that will be moved to the destination if the caller needs it + if (null != newFiles) { + newFiles.add(destFile); + } + taskId++; + } + } catch(IOException e) { + throw new HiveException(e.getMessage(), e); + } + } + /** * Replaces files in the partition with new data set specified by srcf. Works * by renaming directory of srcf to the destination file. @@ -3945,7 +3993,7 @@ private static void moveAcidFiles(String deltaFileType, PathFilter pathFilter, F */ protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, HiveConf conf, boolean isSrcLocal, boolean purge, List newFiles, PathFilter deletePathFilter, - boolean isMmTableOverwrite) throws HiveException { + boolean isMmTableOverwrite, boolean isBucketed) throws HiveException { try { FileSystem destFs = destf.getFileSystem(conf); @@ -3983,29 +4031,14 @@ protected void replaceFiles(Path tablePath, Path srcf, Path destf, Path oldPath, // directory if it is the root of an HDFS encryption zone. // 2. srcs must be a list of files -- ensured by LoadSemanticAnalyzer // in both cases, we move the file under destf + FileStatus[] files; if (srcs.length == 1 && srcs[0].isDirectory()) { - if (!moveFile(conf, srcs[0].getPath(), destf, true, isSrcLocal)) { - throw new IOException("Error moving: " + srcf + " into: " + destf); - } - - // Add file paths of the files that will be moved to the destination if the caller needs it - if (null != newFiles) { - listNewFilesRecursively(destFs, destf, newFiles); - } + files = srcFs.listStatus(srcs[0].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER); } else { // its either a file or glob - for (FileStatus src : srcs) { - Path destFile = new Path(destf, src.getPath().getName()); - if (!moveFile(conf, src.getPath(), destFile, true, isSrcLocal)) { - throw new IOException("Error moving: " + srcf + " into: " + destf); - } - - // Add file paths of the files that will be moved to the destination if the caller needs it - if (null != newFiles) { - newFiles.add(destFile); - } - } + files = srcs; } + moveFiles(files, srcf, destf, newFiles, conf, isSrcLocal, isBucketed); } catch (IOException e) { throw new HiveException(e.getMessage(), e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 4535c3edc2..db5f4add41 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -168,6 +168,50 @@ private URI initializeFromURI(String fromPath, boolean isLocal) throws IOExcepti } } } + // Do another loop if table is bucketed + List bucketCols = table.getBucketCols(); + if (bucketCols != null && !bucketCols.isEmpty()) { + // Hive assumes that user names the files as per the corresponding + // bucket. For e.g, file names should follow the format 000000_0, 000000_1 etc. + // Here the 1st file will belong to bucket 0 and 2nd to bucket 1 and so on. + boolean[] bucketArray = new boolean[table.getNumBuckets()]; + // initialize the array + int numBuckets = table.getNumBuckets(); + for (int i = 0; i < numBuckets; i++) { + bucketArray[i] = false; + } + + for (FileStatus oneSrc : srcs) { + String bucketName = oneSrc.getPath().getName(); + + //get the bucket id + String bucketIdStr = + Utilities.getBucketFileNameFromPathSubString(bucketName); + int bucketId = Utilities.getBucketIdFromFile(bucketIdStr); + LOG.info("bucket ID for file " + oneSrc.getPath() + " = " + bucketId + + " for table " + table.getFullyQualifiedName()); + if (bucketId == -1) { + throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg( + "The file name is invalid : " + + oneSrc.getPath().toString() + " for table " + + table.getFullyQualifiedName())); + } + if (bucketId >= numBuckets) { + throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg( + "The file name corresponds to invalid bucketId : " + + oneSrc.getPath().toString()) + + ". Maximum number of buckets can be " + numBuckets + + " for table " + table.getFullyQualifiedName()); + } + if (bucketArray[bucketId]) { + throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg( + "Multiple files for same bucket : " + bucketId + + ". Only 1 file per bucket allowed in single load command. To load multiple files for same bucket, use multiple statements for table " + + table.getFullyQualifiedName())); + } + bucketArray[bucketId] = true; + } + } } catch (IOException e) { // Has to use full name to make sure it does not conflict with // org.apache.commons.lang.StringUtils diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveCopyFiles.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveCopyFiles.java index cc1d8574b0..a4f2102c54 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveCopyFiles.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveCopyFiles.java @@ -83,7 +83,7 @@ public void testRenameNewFilesOnSameFileSystem() throws IOException { FileSystem targetFs = targetPath.getFileSystem(hiveConf); try { - Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false,null); + Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false,null, false); } catch (HiveException e) { e.printStackTrace(); assertTrue("Hive.copyFiles() threw an unexpected exception.", false); @@ -107,7 +107,7 @@ public void testRenameExistingFilesOnSameFileSystem() throws IOException { FileSystem targetFs = targetPath.getFileSystem(hiveConf); try { - Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null); + Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null, false); } catch (HiveException e) { e.printStackTrace(); assertTrue("Hive.copyFiles() threw an unexpected exception.", false); @@ -127,7 +127,7 @@ public void testRenameExistingFilesOnSameFileSystem() throws IOException { sourceFolder.newFile("000001_0.gz"); try { - Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null); + Hive.copyFiles(hiveConf, sourcePath, targetPath, targetFs, isSourceLocal, NO_ACID, false, null, false); } catch (HiveException e) { e.printStackTrace(); assertTrue("Hive.copyFiles() threw an unexpected exception.", false); @@ -158,7 +158,7 @@ public void testCopyNewFilesOnDifferentFileSystem() throws IOException { Mockito.when(spyTargetFs.getUri()).thenReturn(URI.create("hdfs://" + targetPath.toUri().getPath())); try { - Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null); + Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false); } catch (HiveException e) { e.printStackTrace(); assertTrue("Hive.copyFiles() threw an unexpected exception.", false); @@ -185,7 +185,7 @@ public void testCopyExistingFilesOnDifferentFileSystem() throws IOException { Mockito.when(spyTargetFs.getUri()).thenReturn(URI.create("hdfs://" + targetPath.toUri().getPath())); try { - Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null); + Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false); } catch (HiveException e) { e.printStackTrace(); assertTrue("Hive.copyFiles() threw an unexpected exception.", false); @@ -205,7 +205,7 @@ public void testCopyExistingFilesOnDifferentFileSystem() throws IOException { sourceFolder.newFile("000001_0.gz"); try { - Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null); + Hive.copyFiles(hiveConf, sourcePath, targetPath, spyTargetFs, isSourceLocal, NO_ACID, false, null, false); } catch (HiveException e) { e.printStackTrace(); assertTrue("Hive.copyFiles() threw an unexpected exception.", false); diff --git a/ql/src/test/queries/clientnegative/load_data_bucketed_1.q b/ql/src/test/queries/clientnegative/load_data_bucketed_1.q new file mode 100644 index 0000000000..032171727c --- /dev/null +++ b/ql/src/test/queries/clientnegative/load_data_bucketed_1.q @@ -0,0 +1,7 @@ +set hive.mapred.mode=nonstrict; + +drop table if exists src10; +CREATE TABLE src10(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE; + +load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE src10 partition(ds='2008-04-08'); + diff --git a/ql/src/test/queries/clientnegative/load_data_bucketed_2.q b/ql/src/test/queries/clientnegative/load_data_bucketed_2.q new file mode 100644 index 0000000000..3f836e97c3 --- /dev/null +++ b/ql/src/test/queries/clientnegative/load_data_bucketed_2.q @@ -0,0 +1,7 @@ +set hive.mapred.mode=nonstrict; + +drop table if exists src10; +CREATE TABLE src10(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE; + +load data local inpath '../../data/files/auto_sortmerge_join/small/000002_0' INTO TABLE src10 partition(ds='2008-04-08'); + diff --git a/ql/src/test/queries/clientpositive/load_data_rename.q b/ql/src/test/queries/clientpositive/load_data_rename.q new file mode 100644 index 0000000000..f745c70895 --- /dev/null +++ b/ql/src/test/queries/clientpositive/load_data_rename.q @@ -0,0 +1,10 @@ +set hive.mapred.mode=nonstrict; + +drop table if exists src10; +CREATE TABLE src10 (key int, value string) partitioned by (ds string) STORED AS TEXTFILE; +load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE src10 partition(ds='2008-04-08'); + +-- Loaded file should be renamed to 000000_0 +dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/src10/ds=2008-04-08/000000_0; + +drop table src10; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_7.q b/ql/src/test/queries/clientpositive/smb_mapjoin_7.q index 4a6afb0496..fed931c897 100644 --- a/ql/src/test/queries/clientpositive/smb_mapjoin_7.q +++ b/ql/src/test/queries/clientpositive/smb_mapjoin_7.q @@ -16,8 +16,8 @@ create table smb_join_results(k1 int, v1 string, k2 int, v2 string); create table smb_join_results_empty_bigtable(k1 int, v1 string, k2 int, v2 string); create table normal_join_results(k1 int, v1 string, k2 int, v2 string); -load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1; -load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1; +load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1; +load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1; insert overwrite table smb_bucket4_2 select * from src; diff --git a/ql/src/test/results/clientnegative/load_data_bucketed_1.q.out b/ql/src/test/results/clientnegative/load_data_bucketed_1.q.out new file mode 100644 index 0000000000..d721364a08 --- /dev/null +++ b/ql/src/test/results/clientnegative/load_data_bucketed_1.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: drop table if exists src10 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists src10 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE src10(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10 +POSTHOOK: query: CREATE TABLE src10(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10 +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientnegative/load_data_bucketed_2.q.out b/ql/src/test/results/clientnegative/load_data_bucketed_2.q.out new file mode 100644 index 0000000000..d721364a08 --- /dev/null +++ b/ql/src/test/results/clientnegative/load_data_bucketed_2.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: drop table if exists src10 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists src10 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE src10(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10 +POSTHOOK: query: CREATE TABLE src10(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10 +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out index 7a6f8c53a5..4b1313dc93 100644 --- a/ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out +++ b/ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out @@ -38,19 +38,19 @@ POSTHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2 POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@normal_join_results -PREHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 +PREHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1 PREHOOK: type: LOAD #### A masked pattern was here #### PREHOOK: Output: default@smb_bucket4_1 -POSTHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 +POSTHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket4_1 -PREHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 +PREHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1 PREHOOK: type: LOAD #### A masked pattern was here #### PREHOOK: Output: default@smb_bucket4_1 -POSTHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 +POSTHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket4_1 diff --git a/ql/src/test/results/clientpositive/llap/load_data_rename.q.out b/ql/src/test/results/clientpositive/llap/load_data_rename.q.out new file mode 100644 index 0000000000..dc99488d1b --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/load_data_rename.q.out @@ -0,0 +1,30 @@ +PREHOOK: query: drop table if exists src10 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists src10 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE src10 (key int, value string) partitioned by (ds string) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src10 +POSTHOOK: query: CREATE TABLE src10 (key int, value string) partitioned by (ds string) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src10 +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE src10 partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@src10 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE src10 partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@src10 +POSTHOOK: Output: default@src10@ds=2008-04-08 +#### A masked pattern was here #### +PREHOOK: query: drop table src10 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src10 +PREHOOK: Output: default@src10 +POSTHOOK: query: drop table src10 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src10 +POSTHOOK: Output: default@src10 diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out index b71c5b87c1..83033b07c0 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_7.q.out @@ -38,19 +38,19 @@ POSTHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2 POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@normal_join_results -PREHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 +PREHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1 PREHOOK: type: LOAD #### A masked pattern was here #### PREHOOK: Output: default@smb_bucket4_1 -POSTHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 +POSTHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket4_1 -PREHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 +PREHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1 PREHOOK: type: LOAD #### A masked pattern was here #### PREHOOK: Output: default@smb_bucket4_1 -POSTHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 +POSTHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket4_1 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out index ac49c02913..610abab91b 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out @@ -38,19 +38,19 @@ POSTHOOK: query: create table normal_join_results(k1 int, v1 string, k2 int, v2 POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@normal_join_results -PREHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 +PREHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1 PREHOOK: type: LOAD #### A masked pattern was here #### PREHOOK: Output: default@smb_bucket4_1 -POSTHOOK: query: load data local inpath '../../data/files/empty1.txt' into table smb_bucket4_1 +POSTHOOK: query: load data local inpath '../../data/files/empty/000000_0' into table smb_bucket4_1 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket4_1 -PREHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 +PREHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1 PREHOOK: type: LOAD #### A masked pattern was here #### PREHOOK: Output: default@smb_bucket4_1 -POSTHOOK: query: load data local inpath '../../data/files/empty2.txt' into table smb_bucket4_1 +POSTHOOK: query: load data local inpath '../../data/files/empty/000001_0' into table smb_bucket4_1 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket4_1