From 27b4353e77dbc7625b05103c8d6fa3e3d6bbd1d3 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Mon, 11 Mar 2019 09:57:25 +0100 Subject: [PATCH] PostExecOrcFileDump doesn't work with ACID tables --- .../hadoop/hive/ql/hooks/PostExecOrcFileDump.java | 54 +++++++++++++--------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java index df99674f2c342d638466fd0ff26b0d7ad241b723..87c3db28206f62abb0cef2719fb47e827783bfb3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java @@ -88,28 +88,7 @@ public void run(HookContext hookContext) throws Exception { } for (Path dir : directories) { - FileSystem fs = dir.getFileSystem(conf); - List fileList = HdfsUtils.listLocatedStatus(fs, dir, - hiddenFileFilter); - - for (FileStatus fileStatus : fileList) { - LOG.info("Printing orc file dump for " + fileStatus.getPath()); - if (fileStatus.getLen() > 0) { - try { - // just creating orc reader is going to do sanity checks to make sure its valid ORC file - OrcFile.createReader(fs, fileStatus.getPath()); - console.printError("-- BEGIN ORC FILE DUMP --"); - FileDump.main(new String[]{fileStatus.getPath().toString(), "--rowindex=*"}); - console.printError("-- END ORC FILE DUMP --"); - } catch (FileFormatException e) { - LOG.warn("File " + fileStatus.getPath() + " is not ORC. Skip printing orc file dump"); - } catch (IOException e) { - LOG.warn("Skip printing orc file dump. Exception: " + e.getMessage()); - } - } else { - LOG.warn("Zero length file encountered. Skip printing orc file dump."); - } - } + printFileStatus(console, dir.getFileSystem(conf), dir); } // restore the old out stream @@ -118,4 +97,35 @@ public void run(HookContext hookContext) throws Exception { } } + private void printFileStatus(SessionState.LogHelper console, FileSystem fs, Path dir) throws Exception { + List fileList = HdfsUtils.listLocatedStatus(fs, dir, + hiddenFileFilter); + + for (FileStatus fileStatus : fileList) { + if (fileStatus.isDirectory()) { + + // delta directory in case of ACID + printFileStatus(console, fs, fileStatus.getPath()); + } else { + + LOG.info("Printing orc file dump for " + fileStatus.getPath()); + if (fileStatus.getLen() > 0) { + try { + // just creating orc reader is going to do sanity checks to make sure its valid ORC file + OrcFile.createReader(fs, fileStatus.getPath()); + console.printError("-- BEGIN ORC FILE DUMP --"); + FileDump.main(new String[]{fileStatus.getPath().toString(), "--rowindex=*"}); + console.printError("-- END ORC FILE DUMP --"); + } catch (FileFormatException e) { + LOG.warn("File " + fileStatus.getPath() + " is not ORC. Skip printing orc file dump"); + } catch (IOException e) { + LOG.warn("Skip printing orc file dump. Exception: " + e.getMessage()); + } + } else { + LOG.warn("Zero length file encountered. Skip printing orc file dump."); + } + } + } + } + } -- 2.15.1 (Apple Git-101)