diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 3fb8e76..6ea281c 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -636,6 +636,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_PROTO_EVENTS_TTL("hive.hook.proto.events.ttl", "7d", new TimeValidator(TimeUnit.DAYS), "Time-To-Live (TTL) of proto event files before cleanup."), + HIVE_PROTO_FILE_PER_EVENT("hive.hook.proto.file.per.event", false, + "Whether each proto event has to be written to separate file. " + + "(Use this for FS that does not hflush immediately like S3A)"), // Hadoop Configuration Properties // Properties with null values are ignored and exist only for the purpose of giving us diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java index 673c858..0af30d4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java @@ -186,6 +186,7 @@ private int logFileCount = 0; private ProtoMessageWriter writer; private LocalDate writerDate; + private boolean eventPerFile; EventLogger(HiveConf conf, Clock clock) { this.clock = clock; @@ -197,6 +198,8 @@ LOG.error(ConfVars.HIVE_PROTO_EVENTS_BASE_PATH.varname + " is not set, logging disabled."); } + eventPerFile = conf.getBoolVar(ConfVars.HIVE_PROTO_FILE_PER_EVENT); + LOG.info("Event per file enabled: {}", eventPerFile); DatePartitionedLogger tmpLogger = null; try { if (baseDir != null) { @@ -289,7 +292,16 @@ private void writeEvent(HiveHookEventProto event) { writerDate = logger.getDateFromDir(writer.getPath().getParent().getName()); } writer.writeProto(event); - writer.hflush(); + if (eventPerFile) { + if (writer != null) { + LOG.debug("Event per file enabled. Closing proto event file: {}", writer.getPath()); + IOUtils.closeQuietly(writer); + } + // rollover to next file + writer = logger.getWriter(logFileName + "_" + ++logFileCount); + } else { + writer.hflush(); + } return; } catch (IOException e) { // Something wrong with writer, lets close and reopen.