diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 2cd3c95dc9..54d17df798 100644 --- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper; import org.apache.hadoop.hive.ql.exec.tez.TezJobExecHelper; +import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.processors.CommandProcessor; import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; @@ -760,6 +761,9 @@ public int run(String[] args) throws Exception { ss.updateThreadName(); + // Create views registry + HiveMaterializedViewsRegistry.get().init(); + // execute cli driver work try { return executeDriver(ss, conf, oproc); diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 631c83644a..f9caf83fdc 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2430,6 +2430,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Setting it to 0s disables the timeout."), HIVE_SERVER2_PARALLEL_OPS_IN_SESSION("hive.server2.parallel.ops.in.session", true, "Whether to allow several parallel operations (such as SQL statements) in one session."), + HIVE_SERVER2_DUMMY_MATERIALIZED_VIEWS_REGISTRY("hive.server2.materializedviews.registry.dummy", false, + "Whether the materialized views registry should not cache data and hence forward requests to metastore."), // HiveServer2 WebUI HIVE_SERVER2_WEBUI_BIND_HOST("hive.server2.webui.host", "0.0.0.0", "The host address the HiveServer2 WebUI will listen on"), diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index f79dbac573..a0806948be 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -111,6 +111,7 @@ import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -1121,6 +1122,9 @@ public void init() throws Exception { createRemoteDirs(); } + // Create views registry + HiveMaterializedViewsRegistry.get().init(); + testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); String execEngine = conf.get("hive.execution.engine"); conf.set("hive.execution.engine", "mr"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 7e5c81b6fd..a586155f19 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.calcite.adapter.druid.DruidQuery; @@ -91,8 +92,11 @@ private final ConcurrentMap> materializedViews = new ConcurrentHashMap>(); + /* If this boolean is true, we bypass the cache. */ + private boolean dummy; + /* Whether the cache has been initialized or not. */ - private boolean initialized; + private AtomicBoolean initialized = new AtomicBoolean(false); private HiveMaterializedViewsRegistry() { } @@ -115,10 +119,22 @@ public static HiveMaterializedViewsRegistry get() { * runnable task is created, thus the views will still not be loaded in the cache when * it returns. */ - public void init(final Hive db) { - ExecutorService pool = Executors.newCachedThreadPool(); - pool.submit(new Loader(db)); - pool.shutdown(); + public void init() { + try { + Hive db = Hive.get(); + dummy = db.getConf().getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_DUMMY_MATERIALIZED_VIEWS_REGISTRY); + if (dummy) { + initialized.set(true); + LOG.debug("Dummy materialized views registry has been initialized"); + } else { + // We initialize the cache + ExecutorService pool = Executors.newCachedThreadPool(); + pool.submit(new Loader(db)); + pool.shutdown(); + } + } catch (HiveException e) { + LOG.error("Problem connecting to the metastore when initializing the view registry", e); + } } private class Loader implements Runnable { @@ -136,22 +152,22 @@ public void run() { addMaterializedView(mv, OpType.LOAD); } } - initialized = true; + initialized.set(true); + LOG.debug("Materialized views registry has been initialized"); } catch (HiveException e) { - LOG.error("Problem connecting to the metastore when initializing the view registry"); + LOG.error("Problem connecting to the metastore when initializing the view registry", e); } } } public boolean isInitialized() { - return initialized; + return initialized.get(); } /** * Adds a newly created materialized view to the cache. * * @param materializedViewTable the materialized view - * @param tablesUsed tables used by the materialized view */ public RelOptMaterialization createMaterializedView(Table materializedViewTable) { return addMaterializedView(materializedViewTable, OpType.CREATE); @@ -171,10 +187,13 @@ private RelOptMaterialization addMaterializedView(Table materializedViewTable, O // We are going to create the map for each view in the given database ConcurrentMap cq = new ConcurrentHashMap(); - final ConcurrentMap prevCq = materializedViews.putIfAbsent( - materializedViewTable.getDbName(), cq); - if (prevCq != null) { - cq = prevCq; + if (!dummy) { + // If we are caching the MV, we include it in the cache + final ConcurrentMap prevCq = materializedViews.putIfAbsent( + materializedViewTable.getDbName(), cq); + if (prevCq != null) { + cq = prevCq; + } } // Start the process to add MV to the cache @@ -225,6 +244,10 @@ public void dropMaterializedView(Table materializedViewTable) { * @param tableName the name for the materialized view to remove */ public void dropMaterializedView(String dbName, String tableName) { + if (dummy) { + // Nothing to do + return; + } ConcurrentMap dbMap = materializedViews.get(dbName); if (dbMap != null) { dbMap.remove(tableName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 80c7804dc1..7933b43c28 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -253,7 +253,7 @@ private boolean matchesAny(String string, List matchers) { throws MetaException, InvalidOperationException, UnknownDBException, TException { dbName = dbName.toLowerCase(); - if (SessionState.get().getTempTables().size() == 0) { + if (SessionState.get() == null || SessionState.get().getTempTables().size() == 0) { // No temp tables, just call underlying client return super.getTableObjectsByName(dbName, tableNames); } diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 58b8fb42f2..beac27359d 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -189,7 +189,7 @@ public void run() { initializeWorkloadManagement(hiveConf, sessionHive); // Create views registry - HiveMaterializedViewsRegistry.get().init(sessionHive); + HiveMaterializedViewsRegistry.get().init(); // Setup web UI try {