diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 24c5db0e47..cf3f50ba64 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -86,6 +86,7 @@ private static byte[] confVarByteArray = null; + private static final Map vars = new HashMap(); private static final Map metaConfs = new HashMap(); private final List restrictList = new ArrayList(); @@ -778,13 +779,6 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal METASTORE_TRANSACTIONAL_EVENT_LISTENERS("hive.metastore.transactional.event.listeners", "", "A comma separated list of Java classes that implement the org.apache.hadoop.hive.metastore.MetaStoreEventListener" + " interface. Both the metastore event and corresponding listener method will be invoked in the same JDO transaction."), - NOTIFICATION_SEQUENCE_LOCK_MAX_RETRIES("hive.notification.sequence.lock.max.retries", 5, - "Number of retries required to acquire a lock when getting the next notification sequential ID for entries " - + "in the NOTIFICATION_LOG table."), - NOTIFICATION_SEQUENCE_LOCK_RETRY_SLEEP_INTERVAL("hive.notification.sequence.lock.retry.sleep.interval", 500, - new TimeValidator(TimeUnit.MILLISECONDS), - "Sleep interval between retries to acquire a notification lock as described part of property " - + NOTIFICATION_SEQUENCE_LOCK_MAX_RETRIES.name()), METASTORE_EVENT_DB_LISTENER_TTL("hive.metastore.event.db.listener.timetolive", "86400s", new TimeValidator(TimeUnit.SECONDS), "time after which events will be removed from the database listener queue"), @@ -3273,12 +3267,6 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "llap.daemon.communicator.num.threads"), LLAP_DAEMON_DOWNLOAD_PERMANENT_FNS("hive.llap.daemon.download.permanent.fns", false, "Whether LLAP daemon should localize the resources for permanent UDFs."), - LLAP_TASK_SCHEDULER_AM_REGISTRY_NAME("hive.llap.task.scheduler.am.registry", "llap", - "AM registry name for LLAP task scheduler plugin to register with."), - LLAP_TASK_SCHEDULER_AM_REGISTRY_PRINCIPAL("hive.llap.task.scheduler.am.registry.principal", "", - "The name of the principal used to access ZK AM registry securely."), - LLAP_TASK_SCHEDULER_AM_REGISTRY_KEYTAB_FILE("hive.llap.task.scheduler.am.registry.keytab.file", "", - "The path to the Kerberos keytab file used to access ZK AM registry securely."), LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS( "hive.llap.task.scheduler.node.reenable.min.timeout.ms", "200ms", new TimeValidator(TimeUnit.MILLISECONDS), diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index f8070550e1..68d3cc10fd 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -92,6 +92,8 @@ private static final Logger LOG = LoggerFactory.getLogger(DbNotificationListener.class.getName()); private static CleanerThread cleaner = null; + private static final Object NOTIFICATION_TBL_LOCK = new Object(); + // This is the same object as super.conf, but it's convenient to keep a copy of it as a // HiveConf rather than a Configuration. private HiveConf hiveConf; @@ -571,9 +573,11 @@ private int now() { */ private void process(NotificationEvent event, ListenerEvent listenerEvent) throws MetaException { event.setMessageFormat(msgFactory.getMessageFormat()); - LOG.debug("DbNotificationListener: Processing : {}:{}", event.getEventId(), - event.getMessage()); - HMSHandler.getMSForConf(hiveConf).addNotificationEvent(event); + synchronized (NOTIFICATION_TBL_LOCK) { + LOG.debug("DbNotificationListener: Processing : {}:{}", event.getEventId(), + event.getMessage()); + HMSHandler.getMSForConf(hiveConf).addNotificationEvent(event); + } // Set the DB_NOTIFICATION_EVENT_ID for future reference by other listeners. if (event.isSetEventId()) { @@ -599,7 +603,9 @@ private void process(NotificationEvent event, ListenerEvent listenerEvent) throw @Override public void run() { while (true) { - rs.cleanNotificationEvents(ttl); + synchronized(NOTIFICATION_TBL_LOCK) { + rs.cleanNotificationEvents(ttl); + } LOG.debug("Cleaner thread done"); try { Thread.sleep(sleepTime); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index aae5a7cea4..6a2e4008fc 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -2029,21 +2029,6 @@ public void testViewsReplication() throws IOException { printOutput(driverMirror); run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror); verifyRun("SHOW COLUMNS FROM " + dbName + "_dupe.virtual_view_rename", new String[] {"a", "a_"}, driverMirror); - - // Test "DROP VIEW" - run("DROP VIEW " + dbName + ".virtual_view", driver); - verifyIfTableNotExist(dbName, "virtual_view", metaStoreClient); - - // Perform REPL-DUMP/LOAD - advanceDumpDir(); - run("REPL DUMP " + dbName + " FROM " + incrementalDumpId, driver); - incrementalDumpLocn = getResult(0, 0, driver); - incrementalDumpId = getResult(0, 1, true, driver); - LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId); - run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror); - printOutput(driverMirror); - run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror); - verifyIfTableNotExist(dbName + "_dupe", "virtual_view", metaStoreClientMirror); } @Test @@ -2791,15 +2776,16 @@ public void testConstraints() throws IOException { LOG.info("Dumped to {} with id {}", replDumpLocn, replDumpId); run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror); + // bootstrap replication for constraint is not implemented. Will verify it works once done try { List pks = metaStoreClientMirror.getPrimaryKeys(new PrimaryKeysRequest(dbName+ "_dupe" , "tbl1")); - assertEquals(pks.size(), 1); + assertTrue(pks.isEmpty()); List uks = metaStoreClientMirror.getUniqueConstraints(new UniqueConstraintsRequest(dbName+ "_dupe" , "tbl1")); - assertEquals(uks.size(), 1); + assertTrue(uks.isEmpty()); List fks = metaStoreClientMirror.getForeignKeys(new ForeignKeysRequest(null, null, dbName+ "_dupe" , "tbl2")); - assertEquals(fks.size(), 1); + assertTrue(fks.isEmpty()); List nns = metaStoreClientMirror.getNotNullConstraints(new NotNullConstraintsRequest(dbName+ "_dupe" , "tbl3")); - assertEquals(nns.size(), 1); + assertTrue(nns.isEmpty()); } catch (TException te) { assertNull(te); } diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index d9e760f4b7..663c95bb0b 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -542,7 +542,6 @@ minillaplocal.query.files=\ load_dyn_part5.q,\ lvj_mapjoin.q,\ mapjoin_decimal.q,\ - mapjoin_hint.q,\ mapjoin_emit_interval.q,\ mergejoin_3way.q,\ mrr.q,\ diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java index 83392309bb..65f8f945aa 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java @@ -254,13 +254,13 @@ public DynamicServiceInstanceSet(PathChildrenCache cache, this.instancesCache = cache; this.parent = parent; this.encoder = encoder; - parent.populateCache(instancesCache, false); + parent.populateCache(instancesCache); } @Override public Collection getAll() { - return parent.getAllInternal(); + return parent.getAll(); } @Override @@ -281,12 +281,12 @@ public LlapServiceInstance getInstance(String name) { @Override public Set getByHost(String host) { - return parent.getByHostInternal(host); + return parent.getByHost(host); } @Override public int size() { - return parent.sizeInternal(); + return parent.size(); } @Override @@ -402,9 +402,14 @@ public ApplicationId getApplicationId() throws IOException { } @Override - public void stop() { - CloseableUtils.closeQuietly(slotZnode); + public void start() throws IOException { + super.start(); + } + + @Override + public void stop() throws IOException { super.stop(); + CloseableUtils.closeQuietly(slotZnode); } @Override diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/TezAmInstance.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/TezAmInstance.java deleted file mode 100644 index a71904cf34..0000000000 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/TezAmInstance.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.registry.impl; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.commons.codec.binary.Base64; - -import com.google.common.io.ByteStreams; - -import org.apache.tez.common.security.JobTokenIdentifier; - -import org.apache.hadoop.security.token.Token; - -import java.io.IOException; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.types.AddressTypes; -import org.apache.hadoop.registry.client.types.Endpoint; -import org.apache.hadoop.registry.client.types.ServiceRecord; - -public class TezAmInstance extends ServiceInstanceBase { - private static final Logger LOG = LoggerFactory.getLogger(TezAmInstance.class); - private final int pluginPort; - private Token token; - - public TezAmInstance(ServiceRecord srv) throws IOException { - super(srv, TezAmRegistryImpl.IPC_TEZCLIENT); - final Endpoint plugin = srv.getInternalEndpoint(TezAmRegistryImpl.IPC_PLUGIN); - if (plugin != null) { - this.pluginPort = Integer.parseInt(RegistryTypeUtils.getAddressField( - plugin.addresses.get(0), AddressTypes.ADDRESS_PORT_FIELD)); - } else { - this.pluginPort = -1; - } - } - - public int getPluginPort() { - return pluginPort; - } - - public String getSessionId() { - return getProperties().get(TezAmRegistryImpl.AM_SESSION_ID); - } - - public String getJobIdForPluginToken() { - return getProperties().get(TezAmRegistryImpl.AM_PLUGIN_JOBID); - } - - public Token getPluginToken() { - if (this.token != null) return token; - String tokenString = getProperties().get(TezAmRegistryImpl.AM_PLUGIN_TOKEN); - if (tokenString == null || tokenString.isEmpty()) return null; - byte[] tokenBytes = Base64.decodeBase64(tokenString); - Token token = new Token<>(); - try { - token.readFields(ByteStreams.newDataInput(tokenBytes)); - } catch (IOException e) { - LOG.error("Couldn't read the plugin token from [" + tokenString + "]", e); - return null; - } - this.token = token; - return token; - } -} \ No newline at end of file diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/TezAmRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/TezAmRegistryImpl.java deleted file mode 100644 index 417e571b8d..0000000000 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/TezAmRegistryImpl.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.registry.impl; - -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Collection; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.registry.client.types.Endpoint; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TezAmRegistryImpl extends ZkRegistryBase { - private static final Logger LOG = LoggerFactory.getLogger(TezAmRegistryImpl.class); - - static final String IPC_TEZCLIENT = "tez-client"; - static final String IPC_PLUGIN = "llap-plugin"; - static final String AM_SESSION_ID = "am.session.id", AM_PLUGIN_TOKEN = "am.plugin.token", - AM_PLUGIN_JOBID = "am.plugin.jobid"; - private final static String NAMESPACE_PREFIX = "tez-am-"; - private final static String USER_SCOPE_PATH_PREFIX = "user-"; - private static final String WORKER_PREFIX = "worker-"; - private static final String SASL_LOGIN_CONTEXT_NAME = "TezAmZooKeeperClient"; - - private final String registryName; - - public static TezAmRegistryImpl create(Configuration conf, boolean b) { - String amRegistryName = HiveConf.getVar(conf, ConfVars.LLAP_TASK_SCHEDULER_AM_REGISTRY_NAME); - return StringUtils.isBlank(amRegistryName) ? null - : new TezAmRegistryImpl(amRegistryName, conf, true); - } - - - private TezAmRegistryImpl(String instanceName, Configuration conf, boolean useSecureZk) { - super(instanceName, conf, null, NAMESPACE_PREFIX, USER_SCOPE_PATH_PREFIX, WORKER_PREFIX, - useSecureZk ? SASL_LOGIN_CONTEXT_NAME : null, - HiveConf.getVar(conf, ConfVars.LLAP_TASK_SCHEDULER_AM_REGISTRY_PRINCIPAL), - HiveConf.getVar(conf, ConfVars.LLAP_TASK_SCHEDULER_AM_REGISTRY_KEYTAB_FILE), - null); // Always validate ACLs - this.registryName = instanceName; - LOG.info("AM Zookeeper Registry is enabled with registryid: " + instanceName); - } - - public void initializeWithoutRegistering() throws IOException { - initializeWithoutRegisteringInternal(); - } - - public void populateCache(boolean doInvokeListeners) throws IOException { - PathChildrenCache pcc = ensureInstancesCache(0); - populateCache(pcc, doInvokeListeners); - } - - public String register(int amPort, int pluginPort, String sessionId, - String serializedToken, String jobIdForToken) throws IOException { - ServiceRecord srv = new ServiceRecord(); - Endpoint rpcEndpoint = RegistryTypeUtils.ipcEndpoint( - IPC_TEZCLIENT, new InetSocketAddress(hostname, amPort)); - srv.addInternalEndpoint(rpcEndpoint); - Endpoint pluginEndpoint = null; - if (pluginPort >= 0) { - pluginEndpoint = RegistryTypeUtils.ipcEndpoint( - IPC_PLUGIN, new InetSocketAddress(hostname, pluginPort)); - srv.addInternalEndpoint(pluginEndpoint); - } - srv.set(AM_SESSION_ID, sessionId); - boolean hasToken = serializedToken != null; - srv.set(AM_PLUGIN_TOKEN, hasToken ? serializedToken : ""); - srv.set(AM_PLUGIN_JOBID, jobIdForToken != null ? jobIdForToken : ""); - String uniqueId = registerServiceRecord(srv); - LOG.info("Registered this AM: rpc: {}, plugin: {}, sessionId: {}, token: {}, znodePath: {}", - rpcEndpoint, pluginEndpoint, sessionId, hasToken, getRegistrationZnodePath()); - return uniqueId; - } - - public TezAmInstance getInstance(String name) { - Collection instances = getAllInternal(); - for(TezAmInstance instance : instances) { - if (instance.getWorkerIdentity().equals(name)) { - return instance; - } - } - return null; - } - - @Override - protected TezAmInstance createServiceInstance(ServiceRecord srv) throws IOException { - return new TezAmInstance(srv); - } - - @Override - protected String getZkPathUser(Configuration conf) { - // We assume that AMs and HS2 run under the same user. - return RegistryUtils.currentUser(); - } - - public String getRegistryName() { - return registryName; - } -} diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java index b6773b5e0d..c7737706c6 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java +++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java @@ -58,7 +58,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.zookeeper.KeeperException.InvalidACLException; -import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; @@ -279,27 +278,6 @@ protected final String registerServiceRecord(ServiceRecord srv) throws IOExcepti } - protected final void initializeWithoutRegisteringInternal() throws IOException { - // Create a znode under the rootNamespace parent for this instance of the server - try { - try { - zooKeeperClient.create().creatingParentsIfNeeded().forPath(workersPath); - } catch (NodeExistsException ex) { - // Ignore - this is expected. - } - if (doCheckAcls) { - try { - checkAndSetAcls(); - } catch (Exception ex) { - throw new IOException("Error validating or setting ACLs. " + disableMessage, ex); - } - } - } catch (Exception e) { - LOG.error("Unable to create a parent znode for the registry", e); - throw (e instanceof IOException) ? (IOException)e : new IOException(e); - } - } - private void checkAndSetAcls() throws Exception { if (!UserGroupInformation.isSecurityEnabled()) return; // We are trying to check ACLs on the "workers" directory, which noone except us should be @@ -378,7 +356,7 @@ private void putInCache(String key, Map> cache, instanceSet.add(instance); } - protected final void populateCache(PathChildrenCache instancesCache, boolean doInvokeListeners) { + protected final void populateCache(PathChildrenCache instancesCache) { for (ChildData childData : instancesCache.getCurrentData()) { byte[] data = getWorkerData(childData, workerNodePrefix); if (data == null) continue; @@ -386,11 +364,6 @@ protected final void populateCache(PathChildrenCache instancesCache, boolean doI ServiceRecord srv = encoder.fromBytes(childData.getPath(), data); InstanceType instance = createServiceInstance(srv); addToCache(childData.getPath(), instance.getHost(), instance); - if (doInvokeListeners) { - for (ServiceInstanceStateChangeListener listener : stateChangeListeners) { - listener.onCreate(instance); - } - } } catch (IOException e) { LOG.error("Unable to decode data for zkpath: {}." + " Ignoring from current instances list..", childData.getPath()); @@ -453,12 +426,12 @@ public void childEvent(final CuratorFramework client, final PathChildrenCacheEve // The real implementation for the instanceset... instanceset has its own copy of the // ZK cache yet completely depends on the parent in every other aspect and is thus unneeded. - protected final int sizeInternal() { + public int size() { // not using the path child cache here as there could be more than 1 path per host (worker and slot znodes) return nodeToInstanceCache.size(); } - protected final Set getByHostInternal(String host) { + protected final Set getByHost(String host) { Set byHost = nodeToInstanceCache.get(host); byHost = (byHost == null) ? Sets.newHashSet() : byHost; if (LOG.isDebugEnabled()) { @@ -467,7 +440,7 @@ protected final int sizeInternal() { return byHost; } - protected final Collection getAllInternal() { + protected final Collection getAll() { Set instances = new HashSet<>(); for(Set instanceSet : pathToInstanceCache.values()) { instances.addAll(instanceSet); @@ -560,7 +533,7 @@ public void start() throws IOException { CloseableUtils.class.getName(); } - public void stop() { + public void stop() throws IOException { CloseableUtils.closeQuietly(znode); CloseableUtils.closeQuietly(instancesCache); CloseableUtils.closeQuietly(zooKeeperClient); diff --git a/llap-tez/pom.xml b/llap-tez/pom.xml index 69fbea3ccc..1e5b2354c1 100644 --- a/llap-tez/pom.xml +++ b/llap-tez/pom.xml @@ -46,28 +46,6 @@ - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.curator - apache-curator - ${curator.version} - pom - - - org.apache.curator - curator-test - ${curator.version} - test - - org.apache.commons commons-lang3 ${commons-lang3.version} @@ -89,18 +67,6 @@ - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - true - - - org.apache.hadoop - hadoop-yarn-registry - ${hadoop.version} - true - - org.apache.tez tez-api ${tez.version} diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java index 26747fc5ca..cf8bd469dc 100644 --- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java +++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java @@ -14,13 +14,6 @@ package org.apache.hadoop.hive.llap.tezplugins; -import com.google.common.io.ByteArrayDataOutput; - -import org.apache.hadoop.io.Text; -import org.apache.hadoop.yarn.api.records.ApplicationId; - -import org.apache.hadoop.hive.registry.impl.TezAmRegistryImpl; - import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener; import java.io.IOException; @@ -224,7 +217,6 @@ public void setError(TaskInfo ctx, Throwable t) { private final AtomicInteger assignedTaskCounter = new AtomicInteger(0); private final LlapRegistryService registry = new LlapRegistryService(false); - private final TezAmRegistryImpl amRegistry; private volatile ListenableFuture nodeEnablerFuture; private volatile ListenableFuture delayedTaskSchedulerFuture; @@ -244,8 +236,6 @@ public void setError(TaskInfo ctx, Throwable t) { private int totalGuaranteed = 0, unusedGuaranteed = 0; private LlapTaskCommunicator communicator; - private final int amPort; - private final String serializedToken, jobIdForToken; public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) { this(taskSchedulerContext, new MonotonicClock(), true); @@ -255,13 +245,17 @@ public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) { // The fields that HS2 uses to give AM information about plugin endpoint. /** Whether to enable the endpoint. */ public static final String LLAP_PLUGIN_ENDPOINT_ENABLED = "llap.plugin.endpoint.enabled"; + /** The fake job ID generated by HS2 to use for the job token. */ + public static final String LLAP_PLUGIN_ENDPOINT_JOBID = "llap.plugin.endpoint.jobid"; + /** The job token generated by HS2 for the endpoint. See the comment at the place where + * this is used - the token will be generated by AM after we have AM registry. */ + public static final String LLAP_PLUGIN_ENDPOINT_TOKEN = "llap.plugin.endpoint.token"; @VisibleForTesting public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock clock, boolean initMetrics) { super(taskSchedulerContext); this.clock = clock; - this.amPort = taskSchedulerContext.getAppClientPort(); this.delayedTaskSchedulerCallable = createDelayedTaskSchedulerCallable(); try { this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload()); @@ -273,24 +267,26 @@ public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock if (conf.getBoolean(LLAP_PLUGIN_ENDPOINT_ENABLED, false)) { JobTokenSecretManager sm = null; if (UserGroupInformation.isSecurityEnabled()) { - // Set up the security for plugin endpoint. - // We will create the token and publish it in the AM registry. - // Note: this application ID is bogus and is only needed for JobTokenSecretManager. - ApplicationId id = ApplicationId.newInstance( - System.nanoTime(), (int)(System.nanoTime() % 100000)); - Token token = createAmsToken(id); - serializedToken = serializeToken(token); - jobIdForToken = token.getService().toString(); sm = new JobTokenSecretManager(); - sm.addTokenForJob(jobIdForToken, token); - } else { - serializedToken = jobIdForToken = null; + // Note: for HA/AMRegistry, this token will have to be published in a secure location (ZK). + // Otherwise, when HS2 dies, noone will be able to talk to this AM. + // We could generate it in AM in the first place, but then even the original HS2 + // won't be able to talk to us before we register. Which might be ok. + // For now, there's no mechanism to coordinate this, so we email the token from HS2. + byte[] tokenBytes = Base64.decodeBase64(conf.get(LLAP_PLUGIN_ENDPOINT_TOKEN)); + Token token = new Token<>(); + try { + token.readFields(ByteStreams.newDataInput(tokenBytes)); + } catch (IOException e) { + // This shouldn't really happen on a byte array. + throw new RuntimeException(e); + } + sm.addTokenForJob(conf.get(LLAP_PLUGIN_ENDPOINT_JOBID), token); } pluginEndpoint = new LlapPluginServerImpl(sm, HiveConf.getIntVar(conf, ConfVars.LLAP_PLUGIN_RPC_NUM_HANDLERS), HiveConf.getIntVar(conf, ConfVars.LLAP_PLUGIN_RPC_PORT), this); } else { - serializedToken = jobIdForToken = null; pluginEndpoint = null; } // This is called once per AM, so we don't get the starting duck count here. @@ -362,11 +358,9 @@ public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock } String hostsString = HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS); - LOG.info("Running with configuration: hosts={}, numSchedulableTasksPerNode={}, " - + "nodeBlacklistConf={}, localityConf={}", + LOG.info( + "Running with configuration: hosts={}, numSchedulableTasksPerNode={}, nodeBlacklistConf={}, localityConf={}", hostsString, numSchedulableTasksPerNode, nodeBlacklistConf, localityDelayConf); - this.amRegistry = TezAmRegistryImpl.create(conf, true); - LlapTaskCommunicator peer = LlapTaskCommunicator.instance.get(); if (peer != null) { @@ -378,29 +372,6 @@ public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock } } - private static Token createAmsToken(ApplicationId id) { - if (!UserGroupInformation.isSecurityEnabled()) return null; - JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(id.toString())); - JobTokenSecretManager jobTokenManager = new JobTokenSecretManager(); - Token sessionToken = new Token<>(identifier, jobTokenManager); - sessionToken.setService(identifier.getJobId()); - return sessionToken; - } - - private static String serializeToken(Token token) { - byte[] bytes = null; - try { - ByteArrayDataOutput out = ByteStreams.newDataOutput(); - token.write(out); - bytes = out.toByteArray(); - } catch (IOException e) { - // This shouldn't really happen on a byte array. - throw new RuntimeException(e); - } - return Base64.encodeBase64String(bytes); - } - - @VisibleForTesting void updateGuaranteedCount(int newTotalGuaranteed) { List toUpdate = null; @@ -549,9 +520,6 @@ protected void handleUpdateResult(TaskInfo ti, boolean isOk) { @Override public void initialize() { registry.init(conf); - if (pluginEndpoint != null) { - pluginEndpoint.init(conf); - } } @Override @@ -594,12 +562,6 @@ public Void call() throws Exception { addNode(new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode, metrics), inst); } - if (amRegistry != null) { - amRegistry.start(); - int pluginPort = pluginEndpoint != null ? pluginEndpoint.getActualPort() : -1; - amRegistry.register(amPort, pluginPort, HiveConf.getVar(conf, ConfVars.HIVESESSIONID), - serializedToken, jobIdForToken); - } } finally { writeLock.unlock(); } @@ -720,9 +682,6 @@ public void shutdown() { if (registry != null) { registry.stop(); } - if (amRegistry != null) { - amRegistry.stop(); - } if (pluginEndpoint != null) { pluginEndpoint.stop(); diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/endpoint/LlapPluginServerImpl.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/endpoint/LlapPluginServerImpl.java index 4d5333f995..f3c0d5213f 100644 --- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/endpoint/LlapPluginServerImpl.java +++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/endpoint/LlapPluginServerImpl.java @@ -43,7 +43,6 @@ private final SecretManager secretManager; private final int numHandlers; private final LlapTaskSchedulerService parent; - private final AtomicReference bindAddress = new AtomicReference<>(); public LlapPluginServerImpl(SecretManager secretManager, int numHandlers, int port, LlapTaskSchedulerService parent) { @@ -67,7 +66,7 @@ public void serviceStart() { final Configuration conf = getConfig(); final BlockingService daemonImpl = LlapPluginProtocolProtos.LlapPluginProtocol.newReflectiveBlockingService(this); - server = LlapUtil.startProtocolServer(port, numHandlers, bindAddress , conf, daemonImpl, + server = LlapUtil.startProtocolServer(port, numHandlers, null, conf, daemonImpl, LlapPluginProtocolPB.class, secretManager, new LlapPluginPolicyProvider(), ConfVars.LLAP_PLUGIN_ACL, ConfVars.LLAP_PLUGIN_ACL_DENY); } @@ -78,12 +77,4 @@ public void serviceStop() { server.stop(); } } - - public int getActualPort() { - InetSocketAddress bindAddress = this.bindAddress.get(); - if (bindAddress == null) { - throw new RuntimeException("Cannot get port before the service is started"); - } - return bindAddress.getPort(); - } } diff --git a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java index 51d2e08393..156e62023d 100644 --- a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java +++ b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java @@ -1938,7 +1938,6 @@ public void testDelayedLocalityNodeCommErrorDelayedAllocation() throws IOExcepti nodeDisableTimeoutMillis + "ms"); conf.setBoolean(LlapFixedRegistryImpl.FIXED_REGISTRY_RESOLVE_HOST_NAMES, false); conf.setLong(ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY.varname, localityDelayMs); - conf.set(ConfVars.LLAP_TASK_SCHEDULER_AM_REGISTRY_NAME.varname, ""); doReturn(appAttemptId).when(mockAppCallback).getApplicationAttemptId(); doReturn(11111l).when(mockAppCallback).getCustomClusterIdentifier(); diff --git a/metastore/scripts/upgrade/derby/045-HIVE-16886.derby.sql b/metastore/scripts/upgrade/derby/045-HIVE-16886.derby.sql deleted file mode 100644 index d6a1ce83d5..0000000000 --- a/metastore/scripts/upgrade/derby/045-HIVE-16886.derby.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO "APP"."NOTIFICATION_SEQUENCE" ("NNI_ID", "NEXT_EVENT_ID") SELECT * FROM (VALUES (1,1)) tmp_table WHERE NOT EXISTS ( SELECT "NEXT_EVENT_ID" FROM "APP"."NOTIFICATION_SEQUENCE"); \ No newline at end of file diff --git a/metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql b/metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql index 7b8bef1c09..f4cbba65db 100644 --- a/metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql +++ b/metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql @@ -111,12 +111,6 @@ CREATE TABLE "APP"."KEY_CONSTRAINTS" ("CHILD_CD_ID" BIGINT, "CHILD_INTEGER_IDX" CREATE TABLE "APP"."METASTORE_DB_PROPERTIES" ("PROPERTY_KEY" VARCHAR(255) NOT NULL, "PROPERTY_VALUE" VARCHAR(1000) NOT NULL, "DESCRIPTION" VARCHAR(1000)); -- ---------------------------------------------- --- DML Statements --- ---------------------------------------------- - -INSERT INTO "APP"."NOTIFICATION_SEQUENCE" ("NNI_ID", "NEXT_EVENT_ID") SELECT * FROM (VALUES (1,1)) tmp_table WHERE NOT EXISTS ( SELECT "NEXT_EVENT_ID" FROM "APP"."NOTIFICATION_SEQUENCE"); - --- ---------------------------------------------- -- DDL Statements for indexes -- ---------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql index 756c9c10e6..01b6f908f5 100644 --- a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -3,6 +3,5 @@ RUN '041-HIVE-16556.derby.sql'; RUN '042-HIVE-16575.derby.sql'; RUN '043-HIVE-16922.derby.sql'; RUN '044-HIVE-16997.derby.sql'; -RUN '045-HIVE-16886.derby.sql'; UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; diff --git a/metastore/scripts/upgrade/mssql/030-HIVE-16886.mssql.sql b/metastore/scripts/upgrade/mssql/030-HIVE-16886.mssql.sql deleted file mode 100644 index 0e91a05cc6..0000000000 --- a/metastore/scripts/upgrade/mssql/030-HIVE-16886.mssql.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO NOTIFICATION_SEQUENCE (NNI_ID, NEXT_EVENT_ID) SELECT 1,1 WHERE NOT EXISTS (SELECT NEXT_EVENT_ID FROM NOTIFICATION_SEQUENCE); \ No newline at end of file diff --git a/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql index 3c0169dcea..158fa5a0f0 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-3.0.0.mssql.sql @@ -595,8 +595,6 @@ CREATE TABLE NOTIFICATION_SEQUENCE ALTER TABLE NOTIFICATION_SEQUENCE ADD CONSTRAINT NOTIFICATION_SEQUENCE_PK PRIMARY KEY (NNI_ID); -INSERT INTO NOTIFICATION_SEQUENCE (NNI_ID, NEXT_EVENT_ID) SELECT 1,1 WHERE NOT EXISTS (SELECT NEXT_EVENT_ID FROM NOTIFICATION_SEQUENCE); - -- Constraints for table MASTER_KEYS for class(es) [org.apache.hadoop.hive.metastore.model.MMasterKey] -- Constraints for table IDXS for class(es) [org.apache.hadoop.hive.metastore.model.MIndex] diff --git a/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index cca8426e0d..21d62ae470 100644 --- a/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -4,7 +4,6 @@ SELECT 'Upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; :r 027-HIVE-16575.mssql.sql :r 028-HIVE-16922.mssql.sql :r 029-HIVE-16997.mssql.sql -:r 030-HIVE-16886.mssql.sql UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; diff --git a/metastore/scripts/upgrade/mysql/045-HIVE-16886.mysql.sql b/metastore/scripts/upgrade/mysql/045-HIVE-16886.mysql.sql deleted file mode 100644 index d4d798d5e1..0000000000 --- a/metastore/scripts/upgrade/mysql/045-HIVE-16886.mysql.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO `NOTIFICATION_SEQUENCE` (`NNI_ID`, `NEXT_EVENT_ID`) SELECT 1,1 WHERE NOT EXISTS ( SELECT `NEXT_EVENT_ID` FROM `NOTIFICATION_SEQUENCE`); \ No newline at end of file diff --git a/metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql index 6091801e97..98c107754b 100644 --- a/metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/hive-schema-3.0.0.mysql.sql @@ -812,8 +812,6 @@ CREATE TABLE IF NOT EXISTS `NOTIFICATION_SEQUENCE` PRIMARY KEY (`NNI_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; -INSERT INTO `NOTIFICATION_SEQUENCE` (`NNI_ID`, `NEXT_EVENT_ID`) SELECT 1,1 WHERE NOT EXISTS ( SELECT `NEXT_EVENT_ID` FROM `NOTIFICATION_SEQUENCE`); - CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS` ( `CHILD_CD_ID` BIGINT, diff --git a/metastore/scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index 120a1f88b3..9cd3a62663 100644 --- a/metastore/scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -4,7 +4,6 @@ SOURCE 041-HIVE-16556.mysql.sql; SOURCE 042-HIVE-16575.mysql.sql; SOURCE 043-HIVE-16922.mysql.sql; SOURCE 044-HIVE-16997.mysql.sql; -SOURCE 045-HIVE-16886.mysql.sql; UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; diff --git a/metastore/scripts/upgrade/oracle/045-HIVE-16886.oracle.sql b/metastore/scripts/upgrade/oracle/045-HIVE-16886.oracle.sql deleted file mode 100644 index 9bc9dd50f1..0000000000 --- a/metastore/scripts/upgrade/oracle/045-HIVE-16886.oracle.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO NOTIFICATION_SEQUENCE (NNI_ID, NEXT_EVENT_ID) SELECT 1,1 FROM DUAL WHERE NOT EXISTS ( SELECT NEXT_EVENT_ID FROM NOTIFICATION_SEQUENCE); \ No newline at end of file diff --git a/metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql index 79b9efb741..7b32d782eb 100644 --- a/metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/hive-schema-3.0.0.oracle.sql @@ -576,7 +576,6 @@ CREATE TABLE NOTIFICATION_SEQUENCE ALTER TABLE NOTIFICATION_SEQUENCE ADD CONSTRAINT NOTIFICATION_SEQUENCE_PK PRIMARY KEY (NNI_ID); -INSERT INTO NOTIFICATION_SEQUENCE (NNI_ID, NEXT_EVENT_ID) SELECT 1,1 FROM DUAL WHERE NOT EXISTS ( SELECT NEXT_EVENT_ID FROM NOTIFICATION_SEQUENCE); -- Constraints for table PART_COL_PRIVS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege] diff --git a/metastore/scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index 9b8b162e23..6a266498b5 100644 --- a/metastore/scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -4,7 +4,6 @@ SELECT 'Upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; @042-HIVE-16575.oracle.sql; @043-HIVE-16922.oracle.sql; @044-HIVE-16997.oracle.sql; -@045-HIVE-16886.oracle.sql; UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; diff --git a/metastore/scripts/upgrade/postgres/044-HIVE-16886.postgres.sql b/metastore/scripts/upgrade/postgres/044-HIVE-16886.postgres.sql deleted file mode 100644 index 1c5aea8551..0000000000 --- a/metastore/scripts/upgrade/postgres/044-HIVE-16886.postgres.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO "NOTIFICATION_SEQUENCE" ("NNI_ID", "NEXT_EVENT_ID") SELECT 1,1 WHERE NOT EXISTS ( SELECT "NEXT_EVENT_ID" FROM "NOTIFICATION_SEQUENCE"); \ No newline at end of file diff --git a/metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql index 92a2d38c0e..d2dfdd857e 100644 --- a/metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/hive-schema-3.0.0.postgres.sql @@ -589,8 +589,6 @@ CREATE TABLE "NOTIFICATION_SEQUENCE" PRIMARY KEY ("NNI_ID") ); -INSERT INTO "NOTIFICATION_SEQUENCE" ("NNI_ID", "NEXT_EVENT_ID") SELECT 1,1 WHERE NOT EXISTS ( SELECT "NEXT_EVENT_ID" FROM "NOTIFICATION_SEQUENCE"); - CREATE TABLE "KEY_CONSTRAINTS" ( "CHILD_CD_ID" BIGINT, diff --git a/metastore/scripts/upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index c4d3d5f5d9..ee5a673a72 100644 --- a/metastore/scripts/upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -4,7 +4,6 @@ SELECT 'Upgrading MetaStore schema from 2.3.0 to 3.0.0'; \i 041-HIVE-16575.postgres.sql; \i 042-HIVE-16922.postgres.sql; \i 043-HIVE-16997.postgres.sql; -\i 044-HIVE-16886.postgres.sql; UPDATE "VERSION" SET "SCHEMA_VERSION"='3.0.0', "VERSION_COMMENT"='Hive release version 3.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0'; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index dc1245ebd2..b122c431b2 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -67,8 +67,6 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.model.MConstraint; import org.apache.hadoop.hive.metastore.model.MDatabase; -import org.apache.hadoop.hive.metastore.model.MNotificationLog; -import org.apache.hadoop.hive.metastore.model.MNotificationNextId; import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics; import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics; import org.apache.hadoop.hive.metastore.parser.ExpressionTree; @@ -138,7 +136,7 @@ public MetaStoreDirectSql(PersistenceManager pm, Configuration conf, String sche this.schema = schema; DatabaseProduct dbType = null; try { - dbType = DatabaseProduct.determineDatabaseProduct(getProductName(pm)); + dbType = DatabaseProduct.determineDatabaseProduct(getProductName()); } catch (SQLException e) { LOG.warn("Cannot determine database product; assuming OTHER", e); dbType = DatabaseProduct.OTHER; @@ -192,7 +190,7 @@ public MetaStoreDirectSql(PersistenceManager pm, Configuration conf) { this(pm, conf, ""); } - static String getProductName(PersistenceManager pm) { + private String getProductName() { JDOConnection jdoConn = pm.getDataStoreConnection(); try { return ((Connection)jdoConn.getNativeConnection()).getMetaData().getDatabaseProductName(); @@ -224,17 +222,6 @@ private boolean ensureDbInit() { partColumnQuery = pm.newQuery(MPartitionColumnStatistics.class, "dbName == ''"); partColumnQuery.execute(); - /* - these queries for the notification related tables have to be executed so - that the tables are created. This was not required earlier because we were - interacting with these tables via DataNucleus so it would create them if - they did not exist (mostly used in test, schematool should be used for production). - however this has been changed and we used direct SQL - queries via DataNucleus to interact with them now. - */ - pm.newQuery(MNotificationLog.class, "dbName == ''").execute(); - pm.newQuery(MNotificationNextId.class, "nextEventId < -1").execute(); - return true; } catch (Exception ex) { doCommit = false; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 3053dcb50b..b01e59efbe 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -163,9 +163,6 @@ import org.apache.hadoop.hive.metastore.parser.ExpressionTree; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; -import org.apache.hadoop.hive.metastore.tools.SQLGenerator; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.util.StringUtils; import org.apache.hive.common.util.HiveStringUtils; import org.apache.thrift.TException; @@ -242,7 +239,6 @@ private boolean isInitialized = false; private PersistenceManager pm = null; - private SQLGenerator sqlGenerator = null; private MetaStoreDirectSql directSql = null; private PartitionExpressionProxy expressionProxy = null; private Configuration hiveConf; @@ -431,15 +427,6 @@ private void initializeHelper(Properties dsProps) { LOG.info("ObjectStore, initialize called"); prop = dsProps; pm = getPersistenceManager(); - try { - String productName = MetaStoreDirectSql.getProductName(pm); - sqlGenerator = new SQLGenerator( - DatabaseProduct.determineDatabaseProduct(productName), - new HiveConf(hiveConf, ObjectStore.class)); - } catch (SQLException e) { - LOG.error("error trying to figure out the database product", e); - throw new RuntimeException(e); - } isInitialized = pm != null; if (isInitialized) { expressionProxy = createExpressionProxy(hiveConf); @@ -8251,7 +8238,7 @@ public NotificationEventResponse getNextNotification(NotificationEventRequest rq Query query = null; NotificationEventResponse result = new NotificationEventResponse(); - result.setEvents(new ArrayList<>()); + result.setEvents(new ArrayList()); try { openTransaction(); long lastEvent = rqst.getLastEvent(); @@ -8278,98 +8265,29 @@ public NotificationEventResponse getNextNotification(NotificationEventRequest rq } } - private void lockForUpdate() throws MetaException { - String selectQuery = "select NEXT_EVENT_ID from NOTIFICATION_SEQUENCE"; - String selectForUpdateQuery = sqlGenerator.addForUpdateClause(selectQuery); - new RetryingExecutor(hiveConf, () -> { - Query query = pm.newQuery("javax.jdo.query.SQL", selectForUpdateQuery); - query.setUnique(true); - // only need to execute it to get db Lock - query.execute(); - }).run(); - } - - static class RetryingExecutor { - interface Command { - void process() throws Exception; - } - - private static Logger LOG = LoggerFactory.getLogger(RetryingExecutor.class); - private final int maxRetries; - private final long sleepInterval; - private int currentRetries = 0; - private final Command command; - - RetryingExecutor(Configuration config, Command command) { - this.maxRetries = config.getInt(ConfVars.NOTIFICATION_SEQUENCE_LOCK_MAX_RETRIES.name(), - ConfVars.NOTIFICATION_SEQUENCE_LOCK_MAX_RETRIES.defaultIntVal); - this.sleepInterval = config.getTimeDuration( - ConfVars.NOTIFICATION_SEQUENCE_LOCK_RETRY_SLEEP_INTERVAL.name(), - ConfVars.NOTIFICATION_SEQUENCE_LOCK_RETRY_SLEEP_INTERVAL.defaultLongVal, - TimeUnit.MILLISECONDS - ); - this.command = command; - } - - public void run() throws MetaException { - while (true) { - try { - command.process(); - break; - } catch (Exception e) { - LOG.info( - "Attempting to acquire the DB log notification lock: " + currentRetries + " out of " - + maxRetries + " retries", e); - if (currentRetries >= maxRetries) { - String message = - "Couldn't acquire the DB log notification lock because we reached the maximum" - + " # of retries: {} retries. If this happens too often, then is recommended to " - + "increase the maximum number of retries on the" - + " hive.notification.sequence.lock.max.retries configuration"; - LOG.error(message, e); - throw new MetaException(message + " :: " + e.getMessage()); - } - currentRetries++; - try { - Thread.sleep(sleepInterval); - } catch (InterruptedException e1) { - String msg = "Couldn't acquire the DB notification log lock on " + currentRetries - + " retry, because the following error: "; - LOG.error(msg, e1); - throw new MetaException(msg + e1.getMessage()); - } - } - } - } - } - @Override public void addNotificationEvent(NotificationEvent entry) { boolean commited = false; Query query = null; try { openTransaction(); - lockForUpdate(); - Query objectQuery = pm.newQuery(MNotificationNextId.class); - Collection ids = (Collection) objectQuery.execute(); - MNotificationNextId mNotificationNextId = null; + query = pm.newQuery(MNotificationNextId.class); + Collection ids = (Collection) query.execute(); + MNotificationNextId id = null; boolean needToPersistId; if (ids == null || ids.size() == 0) { - mNotificationNextId = new MNotificationNextId(1L); + id = new MNotificationNextId(1L); needToPersistId = true; } else { - mNotificationNextId = ids.iterator().next(); + id = ids.iterator().next(); needToPersistId = false; } - entry.setEventId(mNotificationNextId.getNextEventId()); - mNotificationNextId.incrementEventId(); - if (needToPersistId) { - pm.makePersistent(mNotificationNextId); - } + entry.setEventId(id.getNextEventId()); + id.incrementEventId(); + if (needToPersistId) + pm.makePersistent(id); pm.makePersistent(translateThriftToDb(entry)); commited = commitTransaction(); - } catch (Exception e) { - LOG.error("couldnot get lock for update", e); } finally { rollbackAndCleanup(commited, query); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java deleted file mode 100644 index 0c0bfefc3e..0000000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package org.apache.hadoop.hive.metastore.tools; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.DatabaseProduct; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Helper class that generates SQL queries with syntax specific to target DB - * todo: why throw MetaException? - */ -@VisibleForTesting -public final class SQLGenerator { - static final private Logger LOG = LoggerFactory.getLogger(SQLGenerator.class.getName()); - private final DatabaseProduct dbProduct; - private final HiveConf conf; - - public SQLGenerator(DatabaseProduct dbProduct, HiveConf conf) { - this.dbProduct = dbProduct; - this.conf = conf; - } - - /** - * Genereates "Insert into T(a,b,c) values(1,2,'f'),(3,4,'c')" for appropriate DB - * - * @param tblColumns e.g. "T(a,b,c)" - * @param rows e.g. list of Strings like 3,4,'d' - * @return fully formed INSERT INTO ... statements - */ - public List createInsertValuesStmt(String tblColumns, List rows) { - if (rows == null || rows.size() == 0) { - return Collections.emptyList(); - } - List insertStmts = new ArrayList<>(); - StringBuilder sb = new StringBuilder(); - switch (dbProduct) { - case ORACLE: - if (rows.size() > 1) { - //http://www.oratable.com/oracle-insert-all/ - //https://livesql.oracle.com/apex/livesql/file/content_BM1LJQ87M5CNIOKPOWPV6ZGR3.html - for (int numRows = 0; numRows < rows.size(); numRows++) { - if (numRows % conf - .getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) { - if (numRows > 0) { - sb.append(" select * from dual"); - insertStmts.add(sb.toString()); - } - sb.setLength(0); - sb.append("insert all "); - } - sb.append("into ").append(tblColumns).append(" values(").append(rows.get(numRows)) - .append(") "); - } - sb.append("select * from dual"); - insertStmts.add(sb.toString()); - return insertStmts; - } - //fall through - case DERBY: - case MYSQL: - case POSTGRES: - case SQLSERVER: - for (int numRows = 0; numRows < rows.size(); numRows++) { - if (numRows % conf - .getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) { - if (numRows > 0) { - insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma - } - sb.setLength(0); - sb.append("insert into ").append(tblColumns).append(" values"); - } - sb.append('(').append(rows.get(numRows)).append("),"); - } - insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma - return insertStmts; - default: - String msg = "Unrecognized database product name <" + dbProduct + ">"; - LOG.error(msg); - throw new IllegalStateException(msg); - } - } - - /** - * Given a {@code selectStatement}, decorated it with FOR UPDATE or semantically equivalent - * construct. If the DB doesn't support, return original select. - */ - public String addForUpdateClause(String selectStatement) throws MetaException { - switch (dbProduct) { - case DERBY: - //https://db.apache.org/derby/docs/10.1/ref/rrefsqlj31783.html - //sadly in Derby, FOR UPDATE doesn't meant what it should - return selectStatement; - case MYSQL: - //http://dev.mysql.com/doc/refman/5.7/en/select.html - case ORACLE: - //https://docs.oracle.com/cd/E17952_01/refman-5.6-en/select.html - case POSTGRES: - //http://www.postgresql.org/docs/9.0/static/sql-select.html - return selectStatement + " for update"; - case SQLSERVER: - //https://msdn.microsoft.com/en-us/library/ms189499.aspx - //https://msdn.microsoft.com/en-us/library/ms187373.aspx - String modifier = " with (updlock)"; - int wherePos = selectStatement.toUpperCase().indexOf(" WHERE "); - if (wherePos < 0) { - return selectStatement + modifier; - } - return selectStatement.substring(0, wherePos) + modifier + - selectStatement.substring(wherePos, selectStatement.length()); - default: - String msg = "Unrecognized database product name <" + dbProduct + ">"; - LOG.error(msg); - throw new MetaException(msg); - } - } - - /** - * Suppose you have a query "select a,b from T" and you want to limit the result set - * to the first 5 rows. The mechanism to do that differs in different DBs. - * Make {@code noSelectsqlQuery} to be "a,b from T" and this method will return the - * appropriately modified row limiting query. - *

- * Note that if {@code noSelectsqlQuery} contains a join, you must make sure that - * all columns are unique for Oracle. - */ - public String addLimitClause(int numRows, String noSelectsqlQuery) throws MetaException { - switch (dbProduct) { - case DERBY: - //http://db.apache.org/derby/docs/10.7/ref/rrefsqljoffsetfetch.html - return "select " + noSelectsqlQuery + " fetch first " + numRows + " rows only"; - case MYSQL: - //http://www.postgresql.org/docs/7.3/static/queries-limit.html - case POSTGRES: - //https://dev.mysql.com/doc/refman/5.0/en/select.html - return "select " + noSelectsqlQuery + " limit " + numRows; - case ORACLE: - //newer versions (12c and later) support OFFSET/FETCH - return "select * from (select " + noSelectsqlQuery + ") where rownum <= " + numRows; - case SQLSERVER: - //newer versions (2012 and later) support OFFSET/FETCH - //https://msdn.microsoft.com/en-us/library/ms189463.aspx - return "select TOP(" + numRows + ") " + noSelectsqlQuery; - default: - String msg = "Unrecognized database product name <" + dbProduct + ">"; - LOG.error(msg); - throw new MetaException(msg); - } - } -} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index f77900d5d7..1887c052be 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hive.metastore.datasource.HikariCPDataSourceProvider; import org.apache.hadoop.hive.metastore.metrics.Metrics; import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; -import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.dbcp.PoolingDataSource; @@ -3563,6 +3562,139 @@ public void releaseLocks() { } } } + /** + * Helper class that generates SQL queries with syntax specific to target DB + * todo: why throw MetaException? + */ + @VisibleForTesting + static final class SQLGenerator { + private final DatabaseProduct dbProduct; + private final HiveConf conf; + SQLGenerator(DatabaseProduct dbProduct, HiveConf conf) { + this.dbProduct = dbProduct; + this.conf = conf; + } + /** + * Genereates "Insert into T(a,b,c) values(1,2,'f'),(3,4,'c')" for appropriate DB + * @param tblColumns e.g. "T(a,b,c)" + * @param rows e.g. list of Strings like 3,4,'d' + * @return fully formed INSERT INTO ... statements + */ + List createInsertValuesStmt(String tblColumns, List rows) { + if(rows == null || rows.size() == 0) { + return Collections.emptyList(); + } + List insertStmts = new ArrayList<>(); + StringBuilder sb = new StringBuilder(); + switch (dbProduct) { + case ORACLE: + if(rows.size() > 1) { + //http://www.oratable.com/oracle-insert-all/ + //https://livesql.oracle.com/apex/livesql/file/content_BM1LJQ87M5CNIOKPOWPV6ZGR3.html + for (int numRows = 0; numRows < rows.size(); numRows++) { + if (numRows % conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) { + if (numRows > 0) { + sb.append(" select * from dual"); + insertStmts.add(sb.toString()); + } + sb.setLength(0); + sb.append("insert all "); + } + sb.append("into ").append(tblColumns).append(" values(").append(rows.get(numRows)).append(") "); + } + sb.append("select * from dual"); + insertStmts.add(sb.toString()); + return insertStmts; + } + //fall through + case DERBY: + case MYSQL: + case POSTGRES: + case SQLSERVER: + for(int numRows = 0; numRows < rows.size(); numRows++) { + if(numRows % conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) { + if(numRows > 0) { + insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma + } + sb.setLength(0); + sb.append("insert into ").append(tblColumns).append(" values"); + } + sb.append('(').append(rows.get(numRows)).append("),"); + } + insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma + return insertStmts; + default: + String msg = "Unrecognized database product name <" + dbProduct + ">"; + LOG.error(msg); + throw new IllegalStateException(msg); + } + } + /** + * Given a {@code selectStatement}, decorated it with FOR UPDATE or semantically equivalent + * construct. If the DB doesn't support, return original select. + */ + String addForUpdateClause(String selectStatement) throws MetaException { + switch (dbProduct) { + case DERBY: + //https://db.apache.org/derby/docs/10.1/ref/rrefsqlj31783.html + //sadly in Derby, FOR UPDATE doesn't meant what it should + return selectStatement; + case MYSQL: + //http://dev.mysql.com/doc/refman/5.7/en/select.html + case ORACLE: + //https://docs.oracle.com/cd/E17952_01/refman-5.6-en/select.html + case POSTGRES: + //http://www.postgresql.org/docs/9.0/static/sql-select.html + return selectStatement + " for update"; + case SQLSERVER: + //https://msdn.microsoft.com/en-us/library/ms189499.aspx + //https://msdn.microsoft.com/en-us/library/ms187373.aspx + String modifier = " with (updlock)"; + int wherePos = selectStatement.toUpperCase().indexOf(" WHERE "); + if(wherePos < 0) { + return selectStatement + modifier; + } + return selectStatement.substring(0, wherePos) + modifier + + selectStatement.substring(wherePos, selectStatement.length()); + default: + String msg = "Unrecognized database product name <" + dbProduct + ">"; + LOG.error(msg); + throw new MetaException(msg); + } + } + /** + * Suppose you have a query "select a,b from T" and you want to limit the result set + * to the first 5 rows. The mechanism to do that differs in different DBs. + * Make {@code noSelectsqlQuery} to be "a,b from T" and this method will return the + * appropriately modified row limiting query. + * + * Note that if {@code noSelectsqlQuery} contains a join, you must make sure that + * all columns are unique for Oracle. + */ + private String addLimitClause(int numRows, String noSelectsqlQuery) throws MetaException { + switch (dbProduct) { + case DERBY: + //http://db.apache.org/derby/docs/10.7/ref/rrefsqljoffsetfetch.html + return "select " + noSelectsqlQuery + " fetch first " + numRows + " rows only"; + case MYSQL: + //http://www.postgresql.org/docs/7.3/static/queries-limit.html + case POSTGRES: + //https://dev.mysql.com/doc/refman/5.0/en/select.html + return "select " + noSelectsqlQuery + " limit " + numRows; + case ORACLE: + //newer versions (12c and later) support OFFSET/FETCH + return "select * from (select " + noSelectsqlQuery + ") where rownum <= " + numRows; + case SQLSERVER: + //newer versions (2012 and later) support OFFSET/FETCH + //https://msdn.microsoft.com/en-us/library/ms189463.aspx + return "select TOP(" + numRows + ") " + noSelectsqlQuery; + default: + String msg = "Unrecognized database product name <" + dbProduct + ">"; + LOG.error(msg); + throw new MetaException(msg); + } + } + } private static class NoPoolConnectionPool implements DataSource { // Note that this depends on the fact that no-one in this class calls anything but diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java index 7882dadd72..5448d0dbaa 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -1,25 +1,29 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.metastore; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import com.codahale.metrics.Counter; -import com.google.common.collect.ImmutableList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.Database; @@ -30,10 +34,10 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.api.NotificationEventRequest; import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.Role; @@ -45,33 +49,19 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.hive.metastore.messaging.EventMessage; import org.apache.hadoop.hive.metastore.metrics.Metrics; import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; -import org.apache.hadoop.hive.metastore.model.MNotificationLog; -import org.apache.hadoop.hive.metastore.model.MNotificationNextId; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; + import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.jdo.Query; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.BrokenBarrierException; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import com.google.common.collect.ImmutableList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import javax.jdo.Query; public class TestObjectStore { private ObjectStore objectStore = null; @@ -214,7 +204,7 @@ public void testTableOps() throws MetaException, InvalidObjectException, NoSuchO StorageDescriptor sd1 = new StorageDescriptor(ImmutableList.of(new FieldSchema("pk_col", "double", null)), "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); - HashMap params = new HashMap<>(); + HashMap params = new HashMap(); params.put("EXTERNAL", "false"); Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd1, null, params, null, null, "MANAGED_TABLE"); objectStore.createTable(tbl1); @@ -283,13 +273,13 @@ public void testPartitionOps() throws MetaException, InvalidObjectException, NoS Database db1 = new Database(DB1, "description", "locationurl", null); objectStore.createDatabase(db1); StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); - HashMap tableParams = new HashMap<>(); + HashMap tableParams = new HashMap(); tableParams.put("EXTERNAL", "false"); FieldSchema partitionKey1 = new FieldSchema("Country", ColumnType.STRING_TYPE_NAME, ""); FieldSchema partitionKey2 = new FieldSchema("State", ColumnType.STRING_TYPE_NAME, ""); Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, null, null, "MANAGED_TABLE"); objectStore.createTable(tbl1); - HashMap partitionParams = new HashMap<>(); + HashMap partitionParams = new HashMap(); partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true"); List value1 = Arrays.asList("US", "CA"); Partition part1 = new Partition(value1, DB1, TABLE1, 111, 111, sd, partitionParams); @@ -370,10 +360,6 @@ public void testDirectSqlErrorMetrics() throws Exception { HiveConf conf = new HiveConf(); conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true); Metrics.initialize(conf); - conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES, - "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " + - "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter" - ); // recall setup so that we get an object store with the metrics initalized setUp(); @@ -411,16 +397,16 @@ protected Database getJdoResult(ObjectStore.GetHelper ctx) throws Meta Assert.assertEquals(1, directSqlErrors.getCount()); } - private static void dropAllStoreObjects(RawStore store) - throws MetaException, InvalidObjectException, InvalidInputException { + public static void dropAllStoreObjects(RawStore store) throws MetaException, InvalidObjectException, InvalidInputException { try { Deadline.registerIfNot(100000); - List functions = store.getAllFunctions(); - for (Function func : functions) { + List funcs = store.getAllFunctions(); + for (Function func : funcs) { store.dropFunction(func.getDbName(), func.getFunctionName()); } List dbs = store.getAllDatabases(); - for (String db : dbs) { + for (int i = 0; i < dbs.size(); i++) { + String db = dbs.get(i); List tbls = store.getAllTables(db); for (String tbl : tbls) { List indexes = store.getIndexes(db, tbl, 100); @@ -473,87 +459,4 @@ public void testQueryCloseOnError() throws Exception { Mockito.verify(spy, Mockito.times(3)) .rollbackAndCleanup(Mockito.anyBoolean(), Mockito.anyObject()); } - - @Ignore( - "This test is here to allow testing with other databases like mysql / postgres etc\n" - + " with user changes to the code. This cannot be run on apache derby because of\n" - + " https://db.apache.org/derby/docs/10.10/devguide/cdevconcepts842385.html" - ) - @Test - public void testConcurrentAddNotifications() throws ExecutionException, InterruptedException { - - final int NUM_THREADS = 10; - CyclicBarrier cyclicBarrier = new CyclicBarrier(NUM_THREADS, - () -> LoggerFactory.getLogger("test") - .debug(NUM_THREADS + " threads going to add notification")); - - HiveConf conf = new HiveConf(); - conf.setVar(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, - MockPartitionExpressionProxy.class.getName()); - /* - Below are the properties that need to be set based on what database this test is going to be run - */ - -// conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "com.mysql.jdbc.Driver"); -// conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, -// "jdbc:mysql://localhost:3306/metastore_db"); -// conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, ""); -// conf.setVar(HiveConf.ConfVars.METASTOREPWD, ""); - - /* - we have to add this one manually as for tests the db is initialized via the metastoreDiretSQL - and we don't run the schema creation sql that includes the an insert for notification_sequence - which can be locked. the entry in notification_sequence happens via notification_event insertion. - */ - objectStore.getPersistenceManager().newQuery(MNotificationLog.class, "eventType==''").execute(); - objectStore.getPersistenceManager().newQuery(MNotificationNextId.class, "nextEventId==-1").execute(); - - objectStore.addNotificationEvent( - new NotificationEvent(0, 0, - EventMessage.EventType.CREATE_DATABASE.toString(), - "CREATE DATABASE DB initial")); - - ExecutorService executorService = Executors.newFixedThreadPool(NUM_THREADS); - for (int i = 0; i < NUM_THREADS; i++) { - final int n = i; - - executorService.execute( - () -> { - ObjectStore store = new ObjectStore(); - store.setConf(conf); - - String eventType = EventMessage.EventType.CREATE_DATABASE.toString(); - NotificationEvent dbEvent = - new NotificationEvent(0, 0, eventType, - "CREATE DATABASE DB" + n); - System.out.println("ADDING NOTIFICATION"); - - try { - cyclicBarrier.await(); - } catch (InterruptedException | BrokenBarrierException e) { - throw new RuntimeException(e); - } - store.addNotificationEvent(dbEvent); - System.out.println("FINISH NOTIFICATION"); - }); - } - executorService.shutdown(); - assertTrue(executorService.awaitTermination(15, TimeUnit.SECONDS)); - - // we have to setup this again as the underlying PMF keeps getting reinitialized with original - // reference closed - ObjectStore store = new ObjectStore(); - store.setConf(conf); - - NotificationEventResponse eventResponse = store.getNextNotification( - new NotificationEventRequest()); - assertEquals(NUM_THREADS + 1, eventResponse.getEventsSize()); - long previousId = 0; - for (NotificationEvent event : eventResponse.getEvents()) { - assertTrue("previous:" + previousId + " current:" + event.getEventId(), - previousId < event.getEventId()); - assertTrue(previousId + 1 == event.getEventId()); - previousId = event.getEventId(); - } - } } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java index 1497c00e5d..4c3b824f83 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java @@ -19,7 +19,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.DatabaseProduct; -import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -151,8 +150,8 @@ private void runAgainstDerby(List queries) throws Exception { @Test public void testSQLGenerator() throws Exception { //teseted on Oracle Database 11g Express Edition Release 11.2.0.2.0 - 64bit Production - SQLGenerator sqlGenerator = - new SQLGenerator(DatabaseProduct.ORACLE, conf); + TxnHandler.SQLGenerator sqlGenerator = + new TxnHandler.SQLGenerator(DatabaseProduct.ORACLE, conf); List rows = new ArrayList<>(); rows.add("'yellow', 1"); List sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows); @@ -174,7 +173,7 @@ public void testSQLGenerator() throws Exception { Assert.assertEquals("Wrong stmt", "insert all into colors(name, category) values('G',997) into colors(name, category) values('G',998) into colors(name, category) values('G',999) select * from dual", sql.get(1)); sqlGenerator = - new SQLGenerator(DatabaseProduct.MYSQL, conf); + new TxnHandler.SQLGenerator(DatabaseProduct.MYSQL, conf); rows.clear(); rows.add("'yellow', 1"); sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows); @@ -193,7 +192,7 @@ public void testSQLGenerator() throws Exception { Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('yellow', 1),('red', 2),('orange', 3),('G',0),('G',1),('G',2),('G',3),('G',4),('G',5),('G',6),('G',7),('G',8),('G',9),('G',10),('G',11),('G',12),('G',13),('G',14),('G',15),('G',16),('G',17),('G',18),('G',19),('G',20),('G',21),('G',22),('G',23),('G',24),('G',25),('G',26),('G',27),('G',28),('G',29),('G',30),('G',31),('G',32),('G',33),('G',34),('G',35),('G',36),('G',37),('G',38),('G',39),('G',40),('G',41),('G',42),('G',43),('G',44),('G',45),('G',46),('G',47),('G',48),('G',49),('G',50),('G',51),('G',52),('G',53),('G',54),('G',55),('G',56),('G',57),('G',58),('G',59),('G',60),('G',61),('G',62),('G',63),('G',64),('G',65),('G',66),('G',67),('G',68),('G',69),('G',70),('G',71),('G',72),('G',73),('G',74),('G',75),('G',76),('G',77),('G',78),('G',79),('G',80),('G',81),('G',82),('G',83),('G',84),('G',85),('G',86),('G',87),('G',88),('G',89),('G',90),('G',91),('G',92),('G',93),('G',94),('G',95),('G',96),('G',97),('G',98),('G',99),('G',100),('G',101),('G',102),('G',103),('G',104),('G',105),('G',106),('G',107),('G',108),('G',109),('G',110),('G',111),('G',112),('G',113),('G',114),('G',115),('G',116),('G',117),('G',118),('G',119),('G',120),('G',121),('G',122),('G',123),('G',124),('G',125),('G',126),('G',127),('G',128),('G',129),('G',130),('G',131),('G',132),('G',133),('G',134),('G',135),('G',136),('G',137),('G',138),('G',139),('G',140),('G',141),('G',142),('G',143),('G',144),('G',145),('G',146),('G',147),('G',148),('G',149),('G',150),('G',151),('G',152),('G',153),('G',154),('G',155),('G',156),('G',157),('G',158),('G',159),('G',160),('G',161),('G',162),('G',163),('G',164),('G',165),('G',166),('G',167),('G',168),('G',169),('G',170),('G',171),('G',172),('G',173),('G',174),('G',175),('G',176),('G',177),('G',178),('G',179),('G',180),('G',181),('G',182),('G',183),('G',184),('G',185),('G',186),('G',187),('G',188),('G',189),('G',190),('G',191),('G',192),('G',193),('G',194),('G',195),('G',196),('G',197),('G',198),('G',199),('G',200),('G',201),('G',202),('G',203),('G',204),('G',205),('G',206),('G',207),('G',208),('G',209),('G',210),('G',211),('G',212),('G',213),('G',214),('G',215),('G',216),('G',217),('G',218),('G',219),('G',220),('G',221),('G',222),('G',223),('G',224),('G',225),('G',226),('G',227),('G',228),('G',229),('G',230),('G',231),('G',232),('G',233),('G',234),('G',235),('G',236),('G',237),('G',238),('G',239),('G',240),('G',241),('G',242),('G',243),('G',244),('G',245),('G',246),('G',247),('G',248),('G',249),('G',250),('G',251),('G',252),('G',253),('G',254),('G',255),('G',256),('G',257),('G',258),('G',259),('G',260),('G',261),('G',262),('G',263),('G',264),('G',265),('G',266),('G',267),('G',268),('G',269),('G',270),('G',271),('G',272),('G',273),('G',274),('G',275),('G',276),('G',277),('G',278),('G',279),('G',280),('G',281),('G',282),('G',283),('G',284),('G',285),('G',286),('G',287),('G',288),('G',289),('G',290),('G',291),('G',292),('G',293),('G',294),('G',295),('G',296),('G',297),('G',298),('G',299),('G',300),('G',301),('G',302),('G',303),('G',304),('G',305),('G',306),('G',307),('G',308),('G',309),('G',310),('G',311),('G',312),('G',313),('G',314),('G',315),('G',316),('G',317),('G',318),('G',319),('G',320),('G',321),('G',322),('G',323),('G',324),('G',325),('G',326),('G',327),('G',328),('G',329),('G',330),('G',331),('G',332),('G',333),('G',334),('G',335),('G',336),('G',337),('G',338),('G',339),('G',340),('G',341),('G',342),('G',343),('G',344),('G',345),('G',346),('G',347),('G',348),('G',349),('G',350),('G',351),('G',352),('G',353),('G',354),('G',355),('G',356),('G',357),('G',358),('G',359),('G',360),('G',361),('G',362),('G',363),('G',364),('G',365),('G',366),('G',367),('G',368),('G',369),('G',370),('G',371),('G',372),('G',373),('G',374),('G',375),('G',376),('G',377),('G',378),('G',379),('G',380),('G',381),('G',382),('G',383),('G',384),('G',385),('G',386),('G',387),('G',388),('G',389),('G',390),('G',391),('G',392),('G',393),('G',394),('G',395),('G',396),('G',397),('G',398),('G',399),('G',400),('G',401),('G',402),('G',403),('G',404),('G',405),('G',406),('G',407),('G',408),('G',409),('G',410),('G',411),('G',412),('G',413),('G',414),('G',415),('G',416),('G',417),('G',418),('G',419),('G',420),('G',421),('G',422),('G',423),('G',424),('G',425),('G',426),('G',427),('G',428),('G',429),('G',430),('G',431),('G',432),('G',433),('G',434),('G',435),('G',436),('G',437),('G',438),('G',439),('G',440),('G',441),('G',442),('G',443),('G',444),('G',445),('G',446),('G',447),('G',448),('G',449),('G',450),('G',451),('G',452),('G',453),('G',454),('G',455),('G',456),('G',457),('G',458),('G',459),('G',460),('G',461),('G',462),('G',463),('G',464),('G',465),('G',466),('G',467),('G',468),('G',469),('G',470),('G',471),('G',472),('G',473),('G',474),('G',475),('G',476),('G',477),('G',478),('G',479),('G',480),('G',481),('G',482),('G',483),('G',484),('G',485),('G',486),('G',487),('G',488),('G',489),('G',490),('G',491),('G',492),('G',493),('G',494),('G',495),('G',496),('G',497),('G',498),('G',499),('G',500),('G',501),('G',502),('G',503),('G',504),('G',505),('G',506),('G',507),('G',508),('G',509),('G',510),('G',511),('G',512),('G',513),('G',514),('G',515),('G',516),('G',517),('G',518),('G',519),('G',520),('G',521),('G',522),('G',523),('G',524),('G',525),('G',526),('G',527),('G',528),('G',529),('G',530),('G',531),('G',532),('G',533),('G',534),('G',535),('G',536),('G',537),('G',538),('G',539),('G',540),('G',541),('G',542),('G',543),('G',544),('G',545),('G',546),('G',547),('G',548),('G',549),('G',550),('G',551),('G',552),('G',553),('G',554),('G',555),('G',556),('G',557),('G',558),('G',559),('G',560),('G',561),('G',562),('G',563),('G',564),('G',565),('G',566),('G',567),('G',568),('G',569),('G',570),('G',571),('G',572),('G',573),('G',574),('G',575),('G',576),('G',577),('G',578),('G',579),('G',580),('G',581),('G',582),('G',583),('G',584),('G',585),('G',586),('G',587),('G',588),('G',589),('G',590),('G',591),('G',592),('G',593),('G',594),('G',595),('G',596),('G',597),('G',598),('G',599),('G',600),('G',601),('G',602),('G',603),('G',604),('G',605),('G',606),('G',607),('G',608),('G',609),('G',610),('G',611),('G',612),('G',613),('G',614),('G',615),('G',616),('G',617),('G',618),('G',619),('G',620),('G',621),('G',622),('G',623),('G',624),('G',625),('G',626),('G',627),('G',628),('G',629),('G',630),('G',631),('G',632),('G',633),('G',634),('G',635),('G',636),('G',637),('G',638),('G',639),('G',640),('G',641),('G',642),('G',643),('G',644),('G',645),('G',646),('G',647),('G',648),('G',649),('G',650),('G',651),('G',652),('G',653),('G',654),('G',655),('G',656),('G',657),('G',658),('G',659),('G',660),('G',661),('G',662),('G',663),('G',664),('G',665),('G',666),('G',667),('G',668),('G',669),('G',670),('G',671),('G',672),('G',673),('G',674),('G',675),('G',676),('G',677),('G',678),('G',679),('G',680),('G',681),('G',682),('G',683),('G',684),('G',685),('G',686),('G',687),('G',688),('G',689),('G',690),('G',691),('G',692),('G',693),('G',694),('G',695),('G',696),('G',697),('G',698),('G',699),('G',700),('G',701),('G',702),('G',703),('G',704),('G',705),('G',706),('G',707),('G',708),('G',709),('G',710),('G',711),('G',712),('G',713),('G',714),('G',715),('G',716),('G',717),('G',718),('G',719),('G',720),('G',721),('G',722),('G',723),('G',724),('G',725),('G',726),('G',727),('G',728),('G',729),('G',730),('G',731),('G',732),('G',733),('G',734),('G',735),('G',736),('G',737),('G',738),('G',739),('G',740),('G',741),('G',742),('G',743),('G',744),('G',745),('G',746),('G',747),('G',748),('G',749),('G',750),('G',751),('G',752),('G',753),('G',754),('G',755),('G',756),('G',757),('G',758),('G',759),('G',760),('G',761),('G',762),('G',763),('G',764),('G',765),('G',766),('G',767),('G',768),('G',769),('G',770),('G',771),('G',772),('G',773),('G',774),('G',775),('G',776),('G',777),('G',778),('G',779),('G',780),('G',781),('G',782),('G',783),('G',784),('G',785),('G',786),('G',787),('G',788),('G',789),('G',790),('G',791),('G',792),('G',793),('G',794),('G',795),('G',796),('G',797),('G',798),('G',799),('G',800),('G',801),('G',802),('G',803),('G',804),('G',805),('G',806),('G',807),('G',808),('G',809),('G',810),('G',811),('G',812),('G',813),('G',814),('G',815),('G',816),('G',817),('G',818),('G',819),('G',820),('G',821),('G',822),('G',823),('G',824),('G',825),('G',826),('G',827),('G',828),('G',829),('G',830),('G',831),('G',832),('G',833),('G',834),('G',835),('G',836),('G',837),('G',838),('G',839),('G',840),('G',841),('G',842),('G',843),('G',844),('G',845),('G',846),('G',847),('G',848),('G',849),('G',850),('G',851),('G',852),('G',853),('G',854),('G',855),('G',856),('G',857),('G',858),('G',859),('G',860),('G',861),('G',862),('G',863),('G',864),('G',865),('G',866),('G',867),('G',868),('G',869),('G',870),('G',871),('G',872),('G',873),('G',874),('G',875),('G',876),('G',877),('G',878),('G',879),('G',880),('G',881),('G',882),('G',883),('G',884),('G',885),('G',886),('G',887),('G',888),('G',889),('G',890),('G',891),('G',892),('G',893),('G',894),('G',895),('G',896),('G',897),('G',898),('G',899),('G',900),('G',901),('G',902),('G',903),('G',904),('G',905),('G',906),('G',907),('G',908),('G',909),('G',910),('G',911),('G',912),('G',913),('G',914),('G',915),('G',916),('G',917),('G',918),('G',919),('G',920),('G',921),('G',922),('G',923),('G',924),('G',925),('G',926),('G',927),('G',928),('G',929),('G',930),('G',931),('G',932),('G',933),('G',934),('G',935),('G',936),('G',937),('G',938),('G',939),('G',940),('G',941),('G',942),('G',943),('G',944),('G',945),('G',946),('G',947),('G',948),('G',949),('G',950),('G',951),('G',952),('G',953),('G',954),('G',955),('G',956),('G',957),('G',958),('G',959),('G',960),('G',961),('G',962),('G',963),('G',964),('G',965),('G',966),('G',967),('G',968),('G',969),('G',970),('G',971),('G',972),('G',973),('G',974),('G',975),('G',976),('G',977),('G',978),('G',979),('G',980),('G',981),('G',982),('G',983),('G',984),('G',985),('G',986),('G',987),('G',988),('G',989),('G',990),('G',991),('G',992),('G',993),('G',994),('G',995),('G',996)", sql.get(0)); Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('G',997),('G',998),('G',999)", sql.get(1)); - sqlGenerator = new SQLGenerator(DatabaseProduct.SQLSERVER, conf); + sqlGenerator = new TxnHandler.SQLGenerator(DatabaseProduct.SQLSERVER, conf); String modSql = sqlGenerator.addForUpdateClause("select nl_next from NEXT_LOCK_ID"); Assert.assertEquals("select nl_next from NEXT_LOCK_ID with (updlock)", modSql); modSql = sqlGenerator.addForUpdateClause("select MT_COMMENT from AUX_TABLE where MT_KEY1='CheckLock' and MT_KEY2=0"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 6da83047b3..b3ef9169c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -562,14 +562,7 @@ "are set. Table schema information is required to read ACID tables"), ACID_TABLES_MUST_BE_READ_WITH_ACID_READER(30021, "An ORC ACID reader required to read ACID tables"), ACID_TABLES_MUST_BE_READ_WITH_HIVEINPUTFORMAT(30022, "Must use HiveInputFormat to read ACID tables " + - "(set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat)"), - - CONCATENATE_UNSUPPORTED_FILE_FORMAT(30030, "Concatenate/Merge only supported for RCFile and ORCFile formats"), - CONCATENATE_UNSUPPORTED_TABLE_BUCKETED(30031, "Concatenate/Merge can not be performed on bucketed tables"), - CONCATENATE_UNSUPPORTED_PARTITION_ARCHIVED(30032, "Concatenate/Merge can not be performed on archived partitions"), - CONCATENATE_UNSUPPORTED_TABLE_NON_NATIVE(30033, "Concatenate/Merge can not be performed on non-native tables"), - CONCATENATE_UNSUPPORTED_TABLE_NOT_MANAGED(30034, "Concatenate/Merge can only be performed on managed tables"), - CONCATENATE_UNSUPPORTED_TABLE_TRANSACTIONAL(30035, "Concatenate/Merge can not be performed on transactional tables") + "(set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat)") ; private int errorCode; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java index 71fb11ff58..dfad6c1929 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java @@ -223,40 +223,16 @@ public void closeOp(boolean abort) throws HiveException { + fss.getLen()); } - Path destDir = finalPath.getParent(); - Path destPath = destDir; // move any incompatible files to final path if (incompatFileSet != null && !incompatFileSet.isEmpty()) { for (Path incompatFile : incompatFileSet) { - // check if path conforms to Hive's file name convention. Hive expects filenames to be in specific format - // like 000000_0, but "LOAD DATA" commands can let you add any files to any partitions/tables without - // renaming. This can cause MoveTask to remove files in some cases where MoveTask assumes the files are - // are generated by speculatively executed tasks. - // Example: MoveTask thinks the following files are same - // part-m-00000_1417075294718 - // part-m-00001_1417075294718 - // Assumes 1417075294718 as taskId and retains only large file supposedly generated by speculative execution. - // This can result in data loss in case of CONCATENATE/merging. Filter out files that does not match Hive's - // filename convention. - if (!Utilities.isHiveManagedFile(incompatFile)) { - // rename un-managed files to conform to Hive's naming standard - // Example: - // /warehouse/table/part-m-00000_1417075294718 will get renamed to /warehouse/table/.hive-staging/000000_0 - // If staging directory already contains the file, taskId_copy_N naming will be used. - final String taskId = Utilities.getTaskId(jc); - Path destFilePath = new Path(destDir, new Path(taskId)); - for (int counter = 1; fs.exists(destFilePath); counter++) { - destFilePath = new Path(destDir, taskId + (Utilities.COPY_KEYWORD + counter)); - } - LOG.warn("Path doesn't conform to Hive's expectation. Renaming {} to {}", incompatFile, destFilePath); - destPath = destFilePath; - } - + Path destDir = finalPath.getParent(); try { - Utilities.renameOrMoveFiles(fs, incompatFile, destPath); - LOG.info("Moved incompatible file " + incompatFile + " to " + destPath); + Utilities.renameOrMoveFiles(fs, incompatFile, destDir); + LOG.info("Moved incompatible file " + incompatFile + " to " + + destDir); } catch (HiveException e) { - LOG.error("Unable to move " + incompatFile + " to " + destPath); + LOG.error("Unable to move " + incompatFile + " to " + destDir); throw new IOException(e); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 8aa2d90b76..646bb23f2e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -79,7 +79,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -4089,22 +4088,13 @@ private int addConstraints(Hive db, AlterTableDesc alterTbl) throws SemanticException, HiveException { try { // This is either an alter table add foreign key or add primary key command. - if (alterTbl.getPrimaryKeyCols() != null && !alterTbl.getPrimaryKeyCols().isEmpty()) { - db.addPrimaryKey(alterTbl.getPrimaryKeyCols()); + if (alterTbl.getForeignKeyCols() != null + && !alterTbl.getForeignKeyCols().isEmpty()) { + db.addForeignKey(alterTbl.getForeignKeyCols()); } - if (alterTbl.getForeignKeyCols() != null && !alterTbl.getForeignKeyCols().isEmpty()) { - try { - db.addForeignKey(alterTbl.getForeignKeyCols()); - } catch (HiveException e) { - if (e.getCause() instanceof InvalidObjectException - && alterTbl.getReplicationSpec()!= null && alterTbl.getReplicationSpec().isInReplicationScope()) { - // During repl load, NoSuchObjectException in foreign key shall - // ignore as the foreign table may not be part of the replication - LOG.debug(e.getMessage()); - } else { - throw e; - } - } + if (alterTbl.getPrimaryKeyCols() != null + && !alterTbl.getPrimaryKeyCols().isEmpty()) { + db.addPrimaryKey(alterTbl.getPrimaryKeyCols()); } if (alterTbl.getUniqueConstraintCols() != null && !alterTbl.getUniqueConstraintCols().isEmpty()) { @@ -4198,7 +4188,7 @@ private void dropPartitions(Hive db, Table tbl, DropTableDesc dropTbl) throws Hi private void dropTable(Hive db, Table tbl, DropTableDesc dropTbl) throws HiveException { // This is a true DROP TABLE - if (tbl != null && dropTbl.getExpectedType() != null) { + if (tbl != null) { if (tbl.isView()) { if (!dropTbl.getExpectView()) { if (dropTbl.getIfExists()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 4322cc6784..aca99f2d83 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -3954,9 +3954,4 @@ public static String getAclStringWithHiveModification(Configuration tezConf, } return aclConf.toAclString(); } - - public static boolean isHiveManagedFile(Path path) { - return AcidUtils.ORIGINAL_PATTERN.matcher(path.getName()).matches() || - AcidUtils.ORIGINAL_PATTERN_COPY.matcher(path.getName()).matches(); - } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index 3cae543dca..7703f31a37 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -18,19 +18,13 @@ Licensed to the Apache Software Foundation (ASF) under one package org.apache.hadoop.hive.ql.exec.repl; import com.google.common.primitives.Ints; - import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.metastore.api.SQLForeignKey; -import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; -import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; -import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.messaging.EventUtils; import org.apache.hadoop.hive.metastore.messaging.MessageFactory; import org.apache.hadoop.hive.metastore.messaging.event.filters.AndFilter; @@ -54,12 +48,11 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; import org.apache.hadoop.hive.ql.parse.repl.dump.events.EventHandler; import org.apache.hadoop.hive.ql.parse.repl.dump.events.EventHandlerFactory; -import org.apache.hadoop.hive.ql.parse.repl.dump.io.ConstraintsSerializer; import org.apache.hadoop.hive.ql.parse.repl.dump.io.FunctionSerializer; import org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; import org.apache.hadoop.hive.ql.parse.repl.dump.log.BootstrapDumpLogger; import org.apache.hadoop.hive.ql.parse.repl.dump.log.IncrementalDumpLogger; -import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,12 +60,10 @@ Licensed to the Apache Software Foundation (ASF) under one import java.io.Serializable; import java.util.Arrays; import java.util.List; -import java.util.UUID; public class ReplDumpTask extends Task implements Serializable { private static final String dumpSchema = "dump_dir,last_repl_id#string,string"; private static final String FUNCTIONS_ROOT_DIR_NAME = "_functions"; - private static final String CONSTRAINTS_ROOT_DIR_NAME = "_constraints"; private static final String FUNCTION_METADATA_FILE_NAME = "_metadata"; private Logger LOG = LoggerFactory.getLogger(ReplDumpTask.class); @@ -201,7 +192,6 @@ private Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws LOG.debug( "analyzeReplDump dumping table: " + tblName + " to db root " + dbRoot.toUri()); dumpTable(dbName, tblName, dbRoot); - dumpConstraintMetadata(dbName, tblName, dbRoot); } Utils.resetDbBootstrapDumpState(hiveDb, dbName, uniqueKey); replLogger.endLog(bootDumpBeginReplId.toString()); @@ -284,7 +274,7 @@ private String getNextDumpDir() { return ReplDumpWork.testInjectDumpDir; } } else { - return UUID.randomUUID().toString(); + return String.valueOf(System.currentTimeMillis()); // TODO: time good enough for now - we'll likely improve this. // We may also work in something the equivalent of pid, thrid and move to nanos to ensure // uniqueness. @@ -310,30 +300,6 @@ private void dumpFunctionMetadata(String dbName, Path dumpRoot) throws Exception } } - private void dumpConstraintMetadata(String dbName, String tblName, Path dbRoot) throws Exception { - try { - Path constraintsRoot = new Path(dbRoot, CONSTRAINTS_ROOT_DIR_NAME); - Path constraintsFile = new Path(constraintsRoot, tblName); - Hive db = getHive(); - List pks = db.getPrimaryKeyList(dbName, tblName); - List fks = db.getForeignKeyList(dbName, tblName); - List uks = db.getUniqueConstraintList(dbName, tblName); - List nns = db.getNotNullConstraintList(dbName, tblName); - if ((pks != null && !pks.isEmpty()) || (fks != null && !fks.isEmpty()) || (uks != null && !uks.isEmpty()) - || (nns != null && !nns.isEmpty())) { - try (JsonWriter jsonWriter = - new JsonWriter(constraintsFile.getFileSystem(conf), constraintsFile)) { - ConstraintsSerializer serializer = new ConstraintsSerializer(pks, fks, uks, nns, conf); - serializer.writeTo(jsonWriter, null); - } - } - } catch (NoSuchObjectException e) { - // Bootstrap constraint dump shouldn't fail if the table is dropped/renamed while dumping it. - // Just log a debug message and skip it. - LOG.debug(e.getMessage()); - } - } - private HiveWrapper.Tuple functionTuple(String functionName, String dbName) { try { HiveWrapper.Tuple tuple = new HiveWrapper(getHive(), dbName).function(functionName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java index 706d0b68be..cd31b173a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadTask.java @@ -25,14 +25,11 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.BootstrapEvent; -import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.ConstraintEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.DatabaseEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.FunctionEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.PartitionEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.TableEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.BootstrapEventsIterator; -import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.ConstraintEventsIterator; -import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadConstraint; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadDatabase; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadFunction; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.TaskTracker; @@ -80,7 +77,6 @@ protected int execute(DriverContext driverContext) { a database ( directory ) */ BootstrapEventsIterator iterator = work.iterator(); - ConstraintEventsIterator constraintIterator = work.constraintIterator(); /* This is used to get hold of a reference during the current creation of tasks and is initialized with "0" tasks such that it will be non consequential in any operations done with task tracker @@ -89,17 +85,8 @@ a database ( directory ) TaskTracker dbTracker = new TaskTracker(ZERO_TASKS); TaskTracker tableTracker = new TaskTracker(ZERO_TASKS); Scope scope = new Scope(); - boolean loadingConstraint = false; - if (!iterator.hasNext() && constraintIterator.hasNext()) { - loadingConstraint = true; - } - while ((iterator.hasNext() || (loadingConstraint && constraintIterator.hasNext())) && loadTaskTracker.canAddMoreTasks()) { - BootstrapEvent next; - if (!loadingConstraint) { - next = iterator.next(); - } else { - next = constraintIterator.next(); - } + while (iterator.hasNext() && loadTaskTracker.canAddMoreTasks()) { + BootstrapEvent next = iterator.next(); switch (next.eventType()) { case Database: DatabaseEvent dbEvent = (DatabaseEvent) next; @@ -186,24 +173,15 @@ a database ( directory ) functionsTracker.debugLog("functions"); break; } - case Constraint: { - LoadConstraint loadConstraint = - new LoadConstraint(context, (ConstraintEvent) next, work.dbNameToLoadIn, dbTracker); - TaskTracker constraintTracker = loadConstraint.tasks(); - scope.rootTasks.addAll(constraintTracker.tasks()); - loadTaskTracker.update(constraintTracker); - constraintTracker.debugLog("constraints"); - } } - if (!loadingConstraint && !iterator.currentDbHasNext()) { + if (!iterator.currentDbHasNext()) { createEndReplLogTask(context, scope, iterator.replLogger()); } } - boolean addAnotherLoadTask = iterator.hasNext() || loadTaskTracker.hasReplicationState() - || constraintIterator.hasNext(); + boolean addAnotherLoadTask = iterator.hasNext() || loadTaskTracker.hasReplicationState(); createBuilderTask(scope.rootTasks, addAnotherLoadTask); - if (!iterator.hasNext() && !constraintIterator.hasNext()) { + if (!iterator.hasNext()) { loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, context, scope)); work.updateDbEventState(null); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java index a8e906750a..f51afe18a1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java @@ -20,7 +20,6 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.DatabaseEvent; import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.BootstrapEventsIterator; -import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem.ConstraintEventsIterator; import org.apache.hadoop.hive.ql.plan.Explain; import java.io.IOException; @@ -33,7 +32,6 @@ Licensed to the Apache Software Foundation (ASF) under one final String dbNameToLoadIn; final String tableNameToLoadIn; private final BootstrapEventsIterator iterator; - private final ConstraintEventsIterator constraintsIterator; private int loadTaskRunCount = 0; private DatabaseEvent.State state = null; @@ -41,7 +39,6 @@ public ReplLoadWork(HiveConf hiveConf, String dumpDirectory, String dbNameToLoad String tableNameToLoadIn) throws IOException { this.tableNameToLoadIn = tableNameToLoadIn; this.iterator = new BootstrapEventsIterator(dumpDirectory, dbNameToLoadIn, hiveConf); - this.constraintsIterator = new ConstraintEventsIterator(dumpDirectory, hiveConf); this.dbNameToLoadIn = dbNameToLoadIn; } @@ -54,10 +51,6 @@ public BootstrapEventsIterator iterator() { return iterator; } - public ConstraintEventsIterator constraintIterator() { - return constraintsIterator; - } - int executedLoadTask() { return ++loadTaskRunCount; } @@ -74,4 +67,4 @@ DatabaseEvent databaseEvent(HiveConf hiveConf) { boolean hasDbState() { return state != null; } -} +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/BootstrapEvent.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/BootstrapEvent.java index 7b7aac963f..db2b0ace95 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/BootstrapEvent.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/BootstrapEvent.java @@ -22,7 +22,7 @@ Licensed to the Apache Software Foundation (ASF) under one EventType eventType(); enum EventType { - Database, Table, Function, Partition, Constraint + Database, Table, Function, Partition } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/ConstraintEvent.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/ConstraintEvent.java deleted file mode 100644 index 7429283935..0000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/ConstraintEvent.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.repl.bootstrap.events; - -import org.apache.hadoop.fs.Path; - -public interface ConstraintEvent extends BootstrapEvent { - Path rootDir(); -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/ConstraintEventsIterator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/ConstraintEventsIterator.java deleted file mode 100644 index 12d4c0d282..0000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/ConstraintEventsIterator.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Iterator; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.parse.EximUtil; -import org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer; - -public class ConstraintEventsIterator implements Iterator { - private FileStatus[] dbDirs; - private int currentDbIndex; - private FileStatus[] constraintFiles = null; - private int currentConstraintIndex; - private FileSystem fs; - private Path path; - - public ConstraintEventsIterator(String dumpDirectory, HiveConf hiveConf) throws IOException { - path = new Path(dumpDirectory); - fs = path.getFileSystem(hiveConf); - } - - private FileStatus[] listConstraintFilesInDBDir(FileSystem fs, Path dbDir) { - try { - return fs.listStatus(new Path(dbDir, ReplicationSemanticAnalyzer.CONSTRAINTS_ROOT_DIR_NAME)); - } catch (FileNotFoundException e) { - return new FileStatus[]{}; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public boolean hasNext() { - if (dbDirs == null) { - try { - dbDirs = fs.listStatus(path, EximUtil.getDirectoryFilter(fs)); - } catch (IOException e) { - throw new RuntimeException(e); - } - currentDbIndex = 0; - if (dbDirs.length != 0) { - currentConstraintIndex = 0; - constraintFiles = listConstraintFilesInDBDir(fs, dbDirs[0].getPath()); - } - } - if ((currentDbIndex < dbDirs.length) && (currentConstraintIndex < constraintFiles.length)) { - return true; - } - while ((currentDbIndex < dbDirs.length) && (currentConstraintIndex == constraintFiles.length)) { - currentDbIndex ++; - if (currentDbIndex < dbDirs.length) { - currentConstraintIndex = 0; - constraintFiles = listConstraintFilesInDBDir(fs, dbDirs[currentDbIndex].getPath()); - } else { - constraintFiles = null; - } - } - return constraintFiles != null; - } - - @Override - public FSConstraintEvent next() { - int thisIndex = currentConstraintIndex; - currentConstraintIndex++; - return new FSConstraintEvent(constraintFiles[thisIndex].getPath()); - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/FSConstraintEvent.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/FSConstraintEvent.java deleted file mode 100644 index a2ad44421d..0000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/FSConstraintEvent.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.filesystem; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.ConstraintEvent; - -public class FSConstraintEvent implements ConstraintEvent { - private final Path rootDir; - - FSConstraintEvent(Path rootDir) { - this.rootDir = rootDir; - } - - @Override - public Path rootDir() { - return rootDir; - } - - @Override - public EventType eventType() { - return EventType.Constraint; - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadConstraint.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadConstraint.java deleted file mode 100644 index fc2aa8d598..0000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadConstraint.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.ErrorMsg; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.ConstraintEvent; -import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.util.Context; -import org.apache.hadoop.hive.ql.parse.EximUtil; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.repl.DumpType; -import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; -import org.apache.hadoop.hive.ql.parse.repl.load.message.AddForeignKeyHandler; -import org.apache.hadoop.hive.ql.parse.repl.load.message.AddNotNullConstraintHandler; -import org.apache.hadoop.hive.ql.parse.repl.load.message.AddPrimaryKeyHandler; -import org.apache.hadoop.hive.ql.parse.repl.load.message.AddUniqueConstraintHandler; -import org.apache.hadoop.hive.ql.parse.repl.load.message.MessageHandler; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.Serializable; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; - -import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.stripQuotes; - -public class LoadConstraint { - private static final Logger LOG = LoggerFactory.getLogger(LoadFunction.class); - private Context context; - private final ConstraintEvent event; - private final String dbNameToLoadIn; - private final TaskTracker tracker; - - public LoadConstraint(Context context, ConstraintEvent event, String dbNameToLoadIn, - TaskTracker existingTracker) { - this.context = context; - this.event = event; - this.dbNameToLoadIn = dbNameToLoadIn; - this.tracker = new TaskTracker(existingTracker); - } - - public TaskTracker tasks() throws IOException, SemanticException { - URI fromURI = EximUtil - .getValidatedURI(context.hiveConf, stripQuotes(event.rootDir().toUri().toString())); - Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(), fromURI.getPath()); - - try { - FileSystem fs = FileSystem.get(fromPath.toUri(), context.hiveConf); - JSONObject json = new JSONObject(EximUtil.readAsString(fs, fromPath)); - String pksString = json.getString("pks"); - String fksString = json.getString("fks"); - String uksString = json.getString("uks"); - String nnsString = json.getString("nns"); - List> tasks = new ArrayList>(); - - AddPrimaryKeyHandler pkHandler = new AddPrimaryKeyHandler(); - DumpMetaData pkDumpMetaData = new DumpMetaData(fromPath, DumpType.EVENT_ADD_PRIMARYKEY, Long.MAX_VALUE, Long.MAX_VALUE, null, - context.hiveConf); - pkDumpMetaData.setPayload(pksString); - tasks.addAll(pkHandler.handle( - new MessageHandler.Context( - dbNameToLoadIn, null, fromPath.toString(), null, pkDumpMetaData, context.hiveConf, - context.hiveDb, null, LOG))); - - AddForeignKeyHandler fkHandler = new AddForeignKeyHandler(); - DumpMetaData fkDumpMetaData = new DumpMetaData(fromPath, DumpType.EVENT_ADD_FOREIGNKEY, Long.MAX_VALUE, Long.MAX_VALUE, null, - context.hiveConf); - fkDumpMetaData.setPayload(fksString); - tasks.addAll(fkHandler.handle( - new MessageHandler.Context( - dbNameToLoadIn, null, fromPath.toString(), null, fkDumpMetaData, context.hiveConf, - context.hiveDb, null, LOG))); - - AddUniqueConstraintHandler ukHandler = new AddUniqueConstraintHandler(); - DumpMetaData ukDumpMetaData = new DumpMetaData(fromPath, DumpType.EVENT_ADD_UNIQUECONSTRAINT, Long.MAX_VALUE, Long.MAX_VALUE, null, - context.hiveConf); - ukDumpMetaData.setPayload(uksString); - tasks.addAll(ukHandler.handle( - new MessageHandler.Context( - dbNameToLoadIn, null, fromPath.toString(), null, ukDumpMetaData, context.hiveConf, - context.hiveDb, null, LOG))); - - AddNotNullConstraintHandler nnHandler = new AddNotNullConstraintHandler(); - DumpMetaData nnDumpMetaData = new DumpMetaData(fromPath, DumpType.EVENT_ADD_NOTNULLCONSTRAINT, Long.MAX_VALUE, Long.MAX_VALUE, null, - context.hiveConf); - nnDumpMetaData.setPayload(nnsString); - tasks.addAll(nnHandler.handle( - new MessageHandler.Context( - dbNameToLoadIn, null, fromPath.toString(), null, nnDumpMetaData, context.hiveConf, - context.hiveDb, null, LOG))); - - tasks.forEach(tracker::addTask); - return tracker; - } catch (Exception e) { - throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); - } - } - -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SessionExpirationTracker.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SessionExpirationTracker.java index da93a3a791..8bee77ea72 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SessionExpirationTracker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SessionExpirationTracker.java @@ -45,7 +45,6 @@ private final long sessionLifetimeMs; private final long sessionLifetimeJitterMs; private final RestartImpl sessionRestartImpl; - private volatile SessionState initSessionState; interface RestartImpl { void closeAndReopenPoolSession(TezSessionPoolSession session) throws Exception; @@ -69,6 +68,7 @@ private SessionExpirationTracker( LOG.debug("Session expiration is enabled; session lifetime is " + sessionLifetimeMs + " + [0, " + sessionLifetimeJitterMs + ") ms"); } + final SessionState initSessionState = SessionState.get(); expirationQueue = new PriorityBlockingQueue<>(11, new Comparator() { @Override public int compare(TezSessionPoolSession o1, TezSessionPoolSession o2) { @@ -179,7 +179,6 @@ private void runExpirationThread() { public void start() { - initSessionState = SessionState.get(); expirationThread.start(); restartThread.start(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPool.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPool.java index 6e8122dc85..4f58565a4c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPool.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPool.java @@ -17,14 +17,6 @@ */ package org.apache.hadoop.hive.ql.exec.tez; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService; - -import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener; -import org.apache.hadoop.hive.registry.impl.TezAmInstance; -import org.apache.hadoop.hive.registry.impl.TezAmRegistryImpl; - import java.io.IOException; import java.util.Queue; import java.util.Set; @@ -56,31 +48,14 @@ private final HiveConf initConf; private final BlockingDeque defaultQueuePool; - private final String amRegistryName; - private final TezAmRegistryImpl amRegistry; - - private final ConcurrentHashMap bySessionId = - new ConcurrentHashMap<>(); - - - TezSessionPool(HiveConf initConf, int numSessionsTotal, boolean useAmRegistryIfPresent) { + TezSessionPool(HiveConf initConf, int numSessionsTotal) { this.initConf = initConf; assert numSessionsTotal > 0; defaultQueuePool = new LinkedBlockingDeque(numSessionsTotal); - this.amRegistry = useAmRegistryIfPresent ? TezAmRegistryImpl.create(initConf, true) : null; - this.amRegistryName = amRegistry == null ? null : amRegistry.getRegistryName(); } void startInitialSessions() throws Exception { if (initialSessions.isEmpty()) return; - if (amRegistry != null) { - amRegistry.start(); - amRegistry.initializeWithoutRegistering(); - // Note: we may later have special logic to pick up old AMs, if any. - amRegistry.registerStateChangeListener(new ChangeListener()); - amRegistry.populateCache(true); - } - int threadCount = Math.min(initialSessions.size(), HiveConf.getIntVar(initConf, ConfVars.HIVE_SERVER2_TEZ_SESSION_MAX_INIT_THREADS)); Preconditions.checkArgument(threadCount > 0); @@ -159,6 +134,7 @@ void replaceSession( // Re-setting the queue config is an old hack that we may remove in future. Path scratchDir = oldSession.getTezScratchDir(); Set additionalFiles = oldSession.getAdditionalFilesNotFromConf(); + HiveConf conf = oldSession.getConf(); String queueName = oldSession.getQueueName(); try { oldSession.close(false); @@ -166,67 +142,24 @@ void replaceSession( if (!wasRemoved) { LOG.error("Old session was closed but it was not in the pool", oldSession); } - bySessionId.remove(oldSession.getSessionId()); } finally { // There's some bogus code that can modify the queue name. Force-set it for pool sessions. // TODO: this might only be applicable to TezSessionPoolManager; try moving it there? - newSession.getConf().set(TezConfiguration.TEZ_QUEUE_NAME, queueName); - // The caller probably created the new session with the old config, but update the - // registry again just in case. TODO: maybe we should enforce that. - configureAmRegistry(newSession); - newSession.open(additionalFiles, scratchDir); + conf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName); + newSession.open(conf, additionalFiles, scratchDir); defaultQueuePool.put(newSession); } } - private void startInitialSession(TezSessionPoolSession session) throws Exception { - boolean isUsable = session.tryUse(); - if (!isUsable) throw new IOException(session + " is not usable at pool startup"); - session.getConf().set(TezConfiguration.TEZ_QUEUE_NAME, session.getQueueName()); - configureAmRegistry(session); - session.open(); - if (session.returnAfterUse()) { - defaultQueuePool.put(session); - } - } - - private void configureAmRegistry(TezSessionPoolSession session) { - if (amRegistryName != null) { - bySessionId.put(session.getSessionId(), session); - HiveConf conf = session.getConf(); - conf.set(ConfVars.LLAP_TASK_SCHEDULER_AM_REGISTRY_NAME.varname, amRegistryName); - conf.set(ConfVars.HIVESESSIONID.varname, session.getSessionId()); - // TODO: can be enable temporarily for testing - // conf.set(LlapTaskSchedulerService.LLAP_PLUGIN_ENDPOINT_ENABLED, "true"); - } - } - - - private final class ChangeListener - implements ServiceInstanceStateChangeListener { - - @Override - public void onCreate(TezAmInstance serviceInstance) { - String sessionId = serviceInstance.getSessionId(); - TezSessionPoolSession session = bySessionId.get(sessionId); - LOG.warn("AM for " + sessionId + " has registered; updating [" + session - + "] with an endpoint at " + serviceInstance.getPluginPort()); - // TODO: actually update the session once WM is committed - } - - @Override - public void onUpdate(TezAmInstance serviceInstance) { - // Presumably we'd get those later if AM updates its stuff. - LOG.warn("Received an unexpected update for instance={}. Ignoring", serviceInstance); - } - - @Override - public void onRemove(TezAmInstance serviceInstance) { - String sessionId = serviceInstance.getSessionId(); - // For now, we don't take any action. In future, we might restore the session based - // on this and get rid of the logic outside of the pool that replaces/reopens/etc. - LOG.warn("AM for " + sessionId + " has disappeared from the registry"); - bySessionId.remove(sessionId); + private void startInitialSession(TezSessionPoolSession sessionState) throws Exception { + HiveConf newConf = new HiveConf(initConf); + // Makes no senses for it to be mixed up like this. + boolean isUsable = sessionState.tryUse(); + if (!isUsable) throw new IOException(sessionState + " is not usable at pool startup"); + newConf.set(TezConfiguration.TEZ_QUEUE_NAME, sessionState.getQueueName()); + sessionState.open(newConf); + if (sessionState.returnAfterUse()) { + defaultQueuePool.put(sessionState); } } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java index 9f721553d6..1f4705c083 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java @@ -114,9 +114,7 @@ public void setupPool(HiveConf conf) throws InterruptedException { int numSessions = conf.getIntVar(ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE); int numSessionsTotal = numSessions * (defaultQueueList.length - emptyNames); if (numSessionsTotal > 0) { - // TODO: this can be enabled to test. Will only be used in WM case for now. - boolean enableAmRegistry = false; - defaultSessionPool = new TezSessionPool(initConf, numSessionsTotal, enableAmRegistry); + defaultSessionPool = new TezSessionPool(initConf, numSessionsTotal); } numConcurrentLlapQueries = conf.getIntVar(ConfVars.HIVE_SERVER2_LLAP_CONCURRENT_QUERIES); @@ -154,16 +152,14 @@ public void setupPool(HiveConf conf) throws InterruptedException { if (queueName.isEmpty()) { continue; } - HiveConf sessionConf = new HiveConf(initConf); - defaultSessionPool.addInitialSession(createAndInitSession(queueName, true, sessionConf)); + defaultSessionPool.addInitialSession(createAndInitSession(queueName, true)); } } } // TODO Create and init session sets up queue, isDefault - but does not initialize the configuration - private TezSessionPoolSession createAndInitSession( - String queue, boolean isDefault, HiveConf conf) { - TezSessionPoolSession sessionState = createSession(TezSessionState.makeSessionId(), conf); + private TezSessionPoolSession createAndInitSession(String queue, boolean isDefault) { + TezSessionPoolSession sessionState = createSession(TezSessionState.makeSessionId()); // TODO When will the queue ever be null. // Pass queue and default in as constructor parameters, and make them final. if (queue != null) { @@ -237,12 +233,12 @@ private TezSessionState getSession(HiveConf conf, boolean doOpen) throws Excepti */ private TezSessionState getNewSessionState(HiveConf conf, String queueName, boolean doOpen) throws Exception { - TezSessionPoolSession retTezSessionState = createAndInitSession(queueName, false, conf); + TezSessionPoolSession retTezSessionState = createAndInitSession(queueName, false); if (queueName != null) { conf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName); } if (doOpen) { - retTezSessionState.open(); + retTezSessionState.open(conf); LOG.info("Started a new session for queue: " + queueName + " session id: " + retTezSessionState.getSessionId()); } @@ -320,8 +316,8 @@ public void destroySession(TezSessionState tezSessionState) throws Exception { tezSessionState.close(false); } - protected TezSessionPoolSession createSession(String sessionId, HiveConf conf) { - return new TezSessionPoolSession(sessionId, this, expirationTracker, conf); + protected TezSessionPoolSession createSession(String sessionId) { + return new TezSessionPoolSession(sessionId, this, expirationTracker); } /* @@ -390,8 +386,13 @@ public TezSessionState getSession( /** Reopens the session that was found to not be running. */ public void reopenSession(TezSessionState sessionState, Configuration conf) throws Exception { HiveConf sessionConf = sessionState.getConf(); - if (sessionState.getQueueName() != null - && sessionConf.get(TezConfiguration.TEZ_QUEUE_NAME) == null) { + // TODO: when will sessionConf be null, other than tests? Set in open. Throw? + if (sessionConf == null) { + LOG.warn("Session configuration is null for " + sessionState); + // default queue name when the initial session was created + sessionConf = new HiveConf(conf, TezSessionPoolManager.class); + } + if (sessionState.getQueueName() != null && sessionConf.get(TezConfiguration.TEZ_QUEUE_NAME) == null) { sessionConf.set(TezConfiguration.TEZ_QUEUE_NAME, sessionState.getQueueName()); } Set oldAdditionalFiles = sessionState.getAdditionalFilesNotFromConf(); @@ -400,7 +401,7 @@ public void reopenSession(TezSessionState sessionState, Configuration conf) thro // Close the old one, but keep the tmp files around. sessionState.close(true); // TODO: should we reuse scratchDir too? - sessionState.open(oldAdditionalFiles, null); + sessionState.open(sessionConf, oldAdditionalFiles, null); } public void closeNonDefaultSessions(boolean keepTmpDir) throws Exception { @@ -421,8 +422,7 @@ public void closeAndReopenPoolSession(TezSessionPoolSession oldSession) throws E if (queueName == null) { LOG.warn("Pool session has a null queue: " + oldSession); } - TezSessionPoolSession newSession = createAndInitSession( - queueName, oldSession.isDefault(), oldSession.getConf()); + TezSessionPoolSession newSession = createAndInitSession(queueName, oldSession.isDefault()); defaultSessionPool.replaceSession(oldSession, newSession); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolSession.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolSession.java index 8ecdbbf999..005eeedc02 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolSession.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolSession.java @@ -56,8 +56,8 @@ private final SessionExpirationTracker expirationTracker; public TezSessionPoolSession(String sessionId, OpenSessionTracker parent, - SessionExpirationTracker expirationTracker, HiveConf conf) { - super(sessionId, conf); + SessionExpirationTracker expirationTracker) { + super(sessionId); this.parent = parent; this.expirationTracker = expirationTracker; } @@ -83,10 +83,10 @@ public void close(boolean keepTmpDir) throws Exception { } @Override - protected void openInternal(Collection additionalFiles, + protected void openInternal(HiveConf conf, Collection additionalFiles, boolean isAsync, LogHelper console, Path scratchDir) throws IOException, LoginException, URISyntaxException, TezException { - super.openInternal(additionalFiles, isAsync, console, scratchDir); + super.openInternal(conf, additionalFiles, isAsync, console, scratchDir); parent.registerOpenSession(this); if (expirationTracker != null) { expirationTracker.addToExpirationQueue(this); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index 170de2143d..fe5c6a1e45 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -93,7 +93,7 @@ private static final String LLAP_LAUNCHER = LlapContainerLauncher.class.getName(); private static final String LLAP_TASK_COMMUNICATOR = LlapTaskCommunicator.class.getName(); - private final HiveConf conf; + private HiveConf conf; private Path tezScratchDir; private LocalResource appJarLr; private TezClient session; @@ -116,9 +116,8 @@ * Constructor. We do not automatically connect, because we only want to * load tez classes when the user has tez installed. */ - public TezSessionState(DagUtils utils, HiveConf conf) { + public TezSessionState(DagUtils utils) { this.utils = utils; - this.conf = conf; } public String toString() { @@ -130,8 +129,8 @@ public String toString() { * Constructor. We do not automatically connect, because we only want to * load tez classes when the user has tez installed. */ - public TezSessionState(String sessionId, HiveConf conf) { - this(DagUtils.getInstance(), conf); + public TezSessionState(String sessionId) { + this(DagUtils.getInstance()); this.sessionId = sessionId; } @@ -177,9 +176,10 @@ public static String makeSessionId() { return UUID.randomUUID().toString(); } - public void open() throws IOException, LoginException, URISyntaxException, TezException { + public void open(HiveConf conf) + throws IOException, LoginException, URISyntaxException, TezException { Set noFiles = null; - open(noFiles, null); + open(conf, noFiles, null); } /** @@ -191,9 +191,9 @@ public void open() throws IOException, LoginException, URISyntaxException, TezEx * @throws TezException * @throws InterruptedException */ - public void open(String[] additionalFiles) - throws IOException, LoginException, URISyntaxException, TezException { - openInternal(setFromArray(additionalFiles), false, null, null); + public void open(HiveConf conf, String[] additionalFiles) + throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException { + openInternal(conf, setFromArray(additionalFiles), false, null, null); } private static Set setFromArray(String[] additionalFiles) { @@ -205,19 +205,20 @@ public void open(String[] additionalFiles) return files; } - public void beginOpen(String[] additionalFiles, LogHelper console) - throws IOException, LoginException, URISyntaxException, TezException { - openInternal(setFromArray(additionalFiles), true, console, null); + public void beginOpen(HiveConf conf, String[] additionalFiles, LogHelper console) + throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException { + openInternal(conf, setFromArray(additionalFiles), true, console, null); } - public void open(Collection additionalFiles, Path scratchDir) + public void open(HiveConf conf, Collection additionalFiles, Path scratchDir) throws LoginException, IOException, URISyntaxException, TezException { - openInternal(additionalFiles, false, null, scratchDir); + openInternal(conf, additionalFiles, false, null, scratchDir); } - protected void openInternal(Collection additionalFiles, - boolean isAsync, LogHelper console, Path scratchDir) - throws IOException, LoginException, URISyntaxException, TezException { + protected void openInternal(final HiveConf conf, Collection additionalFiles, + boolean isAsync, LogHelper console, Path scratchDir) throws IOException, LoginException, + IllegalArgumentException, URISyntaxException, TezException { + this.conf = conf; // TODO Why is the queue name set again. It has already been setup via setQueueName. Do only one of the two. String confQueueName = conf.get(TezConfiguration.TEZ_QUEUE_NAME); if (queueName != null && !queueName.equals(confQueueName)) { @@ -463,7 +464,7 @@ private void setupSessionAcls(Configuration tezConf, HiveConf hiveConf) throws } public void refreshLocalResourcesFromConf(HiveConf conf) - throws IOException, LoginException, URISyntaxException, TezException { + throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException { String dir = tezScratchDir.toString(); @@ -530,6 +531,7 @@ public void close(boolean keepTmpDir) throws Exception { sessionFuture = null; console = null; tezScratchDir = null; + conf = null; appJarLr = null; additionalFilesNotFromConf.clear(); localizedResources.clear(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index e6e236de6e..f1f10286a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -339,9 +339,10 @@ void updateSession(TezSessionState session, TezClient client = session.getSession(); // TODO null can also mean that this operation was interrupted. Should we really try to re-create the session in that case ? if (client == null) { - // Can happen if the user sets the tez flag after the session was established. + // can happen if the user sets the tez flag after the session was + // established LOG.info("Tez session hasn't been created yet. Opening session"); - session.open(inputOutputJars); + session.open(conf, inputOutputJars); } else { LOG.info("Session is already open"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index fa0ba63c25..feacdd8b60 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -132,12 +132,12 @@ private AcidUtils() { } private static final Logger LOG = LoggerFactory.getLogger(AcidUtils.class); - public static final Pattern ORIGINAL_PATTERN = + private static final Pattern ORIGINAL_PATTERN = Pattern.compile("[0-9]+_[0-9]+"); /** * @see org.apache.hadoop.hive.ql.exec.Utilities#COPY_KEYWORD */ - public static final Pattern ORIGINAL_PATTERN_COPY = + private static final Pattern ORIGINAL_PATTERN_COPY = Pattern.compile("[0-9]+_[0-9]+" + COPY_KEYWORD + "[0-9]+"); public static final PathFilter hiddenFileFilter = new PathFilter(){ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java index e28eb34013..b375aea7ff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java @@ -48,7 +48,7 @@ public TailAndFileData(long fileLength, long fileModificationTime, ByteBuffer bb public long fileLength, fileModTime; public int getMemoryUsage() { - return bb.capacity() + 100; // 100 is for 2 longs, BB and java overheads (semi-arbitrary). + return bb.remaining() + 100; // 100 is for 2 longs, BB and java overheads (semi-arbitrary). } } @@ -78,12 +78,8 @@ public void clear() { } public void put(Path path, OrcTail tail) { - ByteBuffer bb = tail.getSerializedTail(); - if (bb.capacity() != bb.remaining()) { - throw new RuntimeException("Bytebuffer allocated for path: " + path + " has remaining: " + bb.remaining() + " != capacity: " + bb.capacity()); - } cache.put(path, new TailAndFileData(tail.getFileTail().getFileLength(), - tail.getFileModificationTime(), bb.duplicate())); + tail.getFileModificationTime(), tail.getSerializedTail().duplicate())); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index aa44c62d88..d661f10c40 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -4203,48 +4203,6 @@ public void dropConstraint(String dbName, String tableName, String constraintNam throws HiveException, NoSuchObjectException { try { getMSC().dropConstraint(dbName, tableName, constraintName); - } catch (NoSuchObjectException e) { - throw e; - } catch (Exception e) { - throw new HiveException(e); - } - } - - public List getPrimaryKeyList(String dbName, String tblName) throws HiveException, NoSuchObjectException { - try { - return getMSC().getPrimaryKeys(new PrimaryKeysRequest(dbName, tblName)); - } catch (NoSuchObjectException e) { - throw e; - } catch (Exception e) { - throw new HiveException(e); - } - } - - public List getForeignKeyList(String dbName, String tblName) throws HiveException, NoSuchObjectException { - try { - return getMSC().getForeignKeys(new ForeignKeysRequest(null, null, dbName, tblName)); - } catch (NoSuchObjectException e) { - throw e; - } catch (Exception e) { - throw new HiveException(e); - } - } - - public List getUniqueConstraintList(String dbName, String tblName) throws HiveException, NoSuchObjectException { - try { - return getMSC().getUniqueConstraints(new UniqueConstraintsRequest(dbName, tblName)); - } catch (NoSuchObjectException e) { - throw e; - } catch (Exception e) { - throw new HiveException(e); - } - } - - public List getNotNullConstraintList(String dbName, String tblName) throws HiveException, NoSuchObjectException { - try { - return getMSC().getNotNullConstraints(new NotNullConstraintsRequest(dbName, tblName)); - } catch (NoSuchObjectException e) { - throw e; } catch (Exception e) { throw new HiveException(e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index 9412e5f417..a2414f3fd3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -103,9 +103,7 @@ joinOp.getConf().setMemoryMonitorInfo(memoryMonitorInfo); TezBucketJoinProcCtx tezBucketJoinProcCtx = new TezBucketJoinProcCtx(context.conf); - boolean hiveConvertJoin = context.conf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN) & - !context.parseContext.getDisableMapJoin(); - if (!hiveConvertJoin) { + if (!context.conf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN)) { // we are just converting to a common merge join operator. The shuffle // join in map-reduce case. Object retval = checkAndConvertSMBJoin(context, joinOp, tezBucketJoinProcCtx, maxSize); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java index 423913b56b..458e8b3291 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.GroupByOperator; +import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.LimitOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorUtils; @@ -52,6 +53,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ColumnStatsList; +import org.apache.hadoop.hive.ql.parse.JoinType; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; @@ -1844,9 +1846,37 @@ private float getSelectivityComplexTree(Operator op) { // No need for overflow checks, assume selectivity is always <= 1.0 float selMultiParent = 1.0f; - for(Operator parent : multiParentOp.getParentOperators()) { - // In the above example, TS-1 -> RS-1 and TS-2 -> RS-2 are simple trees - selMultiParent *= getSelectivitySimpleTree(parent); + + boolean isSelComputed = false; + + // if it is two way left outer or right outer join take selectivity only for + // corresponding branch since only that branch will factor is the reduction + if(multiParentOp instanceof JoinOperator) { + JoinOperator jop = ((JoinOperator)multiParentOp); + isSelComputed = true; + // check for two way join + if(jop.getConf().getConds().length == 1) { + switch(jop.getConf().getCondsList().get(0).getType()) { + case JoinDesc.LEFT_OUTER_JOIN: + selMultiParent *= getSelectivitySimpleTree(multiParentOp.getParentOperators().get(0)); + break; + case JoinDesc.RIGHT_OUTER_JOIN: + selMultiParent *= getSelectivitySimpleTree(multiParentOp.getParentOperators().get(1)); + break; + default: + // for rest of the join type we will take min of the reduction. + float selMultiParentLeft = getSelectivitySimpleTree(multiParentOp.getParentOperators().get(0)); + float selMultiParentRight = getSelectivitySimpleTree(multiParentOp.getParentOperators().get(1)); + selMultiParent = Math.min(selMultiParentLeft, selMultiParentRight); + } + } + } + + if(!isSelComputed) { + for (Operator parent : multiParentOp.getParentOperators()) { + // In the above example, TS-1 -> RS-1 and TS-2 -> RS-2 are simple trees + selMultiParent *= getSelectivitySimpleTree(parent); + } } float selCurrOp = ((float) currentOp.getStatistics().getNumRows() / diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 251decac9b..230ca47e4a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -23,7 +23,6 @@ import org.antlr.runtime.tree.CommonTree; import org.antlr.runtime.tree.Tree; -import org.apache.hadoop.hive.ql.io.AcidUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileSystem; @@ -1674,35 +1673,23 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, // throw a HiveException for other than rcfile and orcfile. if (!((inputFormatClass.equals(RCFileInputFormat.class) || (inputFormatClass.equals(OrcInputFormat.class))))) { - throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_FILE_FORMAT.getMsg()); + throw new SemanticException( + "Only RCFile and ORCFile Formats are supported right now."); } mergeDesc.setInputFormatClass(inputFormatClass); // throw a HiveException if the table/partition is bucketized if (bucketCols != null && bucketCols.size() > 0) { - throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_BUCKETED.getMsg()); + throw new SemanticException( + "Merge can not perform on bucketized partition/table."); } // throw a HiveException if the table/partition is archived if (isArchived) { - throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_PARTITION_ARCHIVED.getMsg()); - } - - // non-native and non-managed tables are not supported as MoveTask requires filenames to be in specific format, - // violating which can cause data loss - if (tblObj.isNonNative()) { - throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_NON_NATIVE.getMsg()); - } - - if (tblObj.getTableType() != TableType.MANAGED_TABLE) { - throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_NOT_MANAGED.getMsg()); + throw new SemanticException( + "Merge can not perform on archived partitions."); } - // transactional tables are compacted and no longer needs to be bucketed, so not safe for merge/concatenation - boolean isAcid = AcidUtils.isAcidTable(tblObj); - if (isAcid) { - throw new SemanticException(ErrorMsg.CONCATENATE_UNSUPPORTED_TABLE_TRANSACTIONAL.getMsg()); - } inputDir.add(oldTblPartLoc); mergeDesc.setInputDir(inputDir); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index ece5495d28..76331fcb81 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -287,7 +287,7 @@ public static MetaData readMetaData(FileSystem fs, Path metadataPath) } } - public static String readAsString(final FileSystem fs, final Path fromMetadataPath) + private static String readAsString(final FileSystem fs, final Path fromMetadataPath) throws IOException { try (FSDataInputStream stream = fs.open(fromMetadataPath)) { byte[] buffer = new byte[1024]; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index 9ab42f2865..adf7f84b65 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -133,8 +133,6 @@ new HashMap<>(); private Map> semiJoinHints; - private boolean disableMapJoin; - public ParseContext() { } @@ -707,12 +705,4 @@ public void setSemiJoinHints(Map> hints) { public Map> getSemiJoinHints() { return semiJoinHints; } - - public void setDisableMapJoin(boolean disableMapJoin) { - this.disableMapJoin = disableMapJoin; - } - - public boolean getDisableMapJoin() { - return disableMapJoin; - } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index 8f6316dacc..7794d3e3ad 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -78,7 +78,6 @@ Licensed to the Apache Software Foundation (ASF) under one private static final String dumpSchema = "dump_dir,last_repl_id#string,string"; public static final String FUNCTIONS_ROOT_DIR_NAME = "_functions"; - public static final String CONSTRAINTS_ROOT_DIR_NAME = "_constraints"; ReplicationSemanticAnalyzer(QueryState queryState) throws SemanticException { super(queryState); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 1c74779dec..cebb0afef9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -9118,31 +9118,6 @@ private int parseSingleSemiJoinHint(Tree args, int curIdx, Map hints) throws SemanticException { - if (hints == null || hints.size() == 0) return false; - for (ASTNode hintNode : hints) { - for (Node node : hintNode.getChildren()) { - ASTNode hint = (ASTNode) node; - if (hint.getChild(0).getType() != HintParser.TOK_MAPJOIN) continue; - Tree args = hint.getChild(1); - if (args.getChildCount() == 1) { - String text = args.getChild(0).getText(); - if (text.equalsIgnoreCase("None")) { - // Hint to disable mapjoin. - return true; - } - } - } - } - return false; - } - - /** * Merges node to target */ private void mergeJoins(QB qb, QBJoinTree node, QBJoinTree target, int pos, int[] tgtToNodeExprMap) { @@ -11402,8 +11377,6 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce // Set the semijoin hints in parse context pCtx.setSemiJoinHints(parseSemiJoinHint(getQB().getParseInfo().getHintList())); - // Set the mapjoin hint if it needs to be disabled. - pCtx.setDisableMapJoin(disableMapJoinWithHint(getQB().getParseInfo().getHintList())); // 5. Take care of view creation if (createVwDesc != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ConstraintsSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ConstraintsSerializer.java deleted file mode 100644 index 2ae9f58d8f..0000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ConstraintsSerializer.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.parse.repl.dump.io; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.SQLForeignKey; -import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; -import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; -import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; -import org.apache.hadoop.hive.metastore.messaging.MessageFactory; -import org.apache.hadoop.hive.ql.parse.ReplicationSpec; -import org.apache.hadoop.hive.ql.parse.SemanticException; - -public class ConstraintsSerializer implements JsonWriter.Serializer { - private HiveConf hiveConf; - private List pks; - private List fks; - private List uks; - private List nns; - - public ConstraintsSerializer(List pks, List fks, - List uks, List nns, HiveConf hiveConf) { - this.hiveConf = hiveConf; - this.pks = pks; - this.fks = fks; - this.uks = uks; - this.nns = nns; - } - - @Override - public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider) - throws SemanticException, IOException { - String pksString, fksString, uksString, nnsString; - pksString = fksString = uksString = nnsString = ""; - if (pks != null) { - pksString = MessageFactory.getInstance().buildAddPrimaryKeyMessage(pks).toString(); - } - if (fks != null) { - fksString = MessageFactory.getInstance().buildAddForeignKeyMessage(fks).toString(); - } - if (uks != null) { - uksString = MessageFactory.getInstance().buildAddUniqueConstraintMessage(uks).toString(); - } - if (nns != null) { - nnsString = MessageFactory.getInstance().buildAddNotNullConstraintMessage(nns).toString(); - } - writer.jsonGenerator.writeStringField("pks", pksString); - writer.jsonGenerator.writeStringField("fks", fksString); - writer.jsonGenerator.writeStringField("uks", uksString); - writer.jsonGenerator.writeStringField("nns", nnsString); - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java index 0fd970ae24..0873c1c1d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java @@ -49,27 +49,19 @@ } } - List> tasks = new ArrayList>(); - if (fks.isEmpty()) { - return tasks; - } - String actualDbName = context.isDbNameEmpty() ? fks.get(0).getFktable_db() : context.dbName; - String actualTblName = context.isTableNameEmpty() ? fks.get(0).getFktable_name() : context.tableName; + String actualTblName = context.isTableNameEmpty() ? fks.get(0).getPktable_name() : context.tableName; for (SQLForeignKey fk : fks) { - // If parent table is in the same database, change it to the actual db on destination - // Otherwise, keep db name - if (fk.getPktable_db().equals(fk.getFktable_db())) { - fk.setPktable_db(actualDbName); - } + fk.setPktable_db(actualDbName); + fk.setPktable_name(actualTblName); fk.setFktable_db(actualDbName); - fk.setFktable_name(actualTblName); } AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList(), fks, new ArrayList(), context.eventOnlyReplicationSpec()); Task addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf); + List> tasks = new ArrayList>(); tasks.add(addConstraintsTask); context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName); updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java index 9c12e7e2af..76cbe5ab9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage; +import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -50,11 +51,6 @@ } } - List> tasks = new ArrayList>(); - if (nns.isEmpty()) { - return tasks; - } - String actualDbName = context.isDbNameEmpty() ? nns.get(0).getTable_db() : context.dbName; String actualTblName = context.isTableNameEmpty() ? nns.get(0).getTable_name() : context.tableName; @@ -66,6 +62,7 @@ AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList(), new ArrayList(), new ArrayList(), nns, context.eventOnlyReplicationSpec()); Task addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf); + List> tasks = new ArrayList>(); tasks.add(addConstraintsTask); context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName); updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java index d7ee223154..aee46dad9b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java @@ -48,12 +48,6 @@ throw (SemanticException)e; } } - - List> tasks = new ArrayList>(); - if (pks.isEmpty()) { - return tasks; - } - String actualDbName = context.isDbNameEmpty() ? pks.get(0).getTable_db() : context.dbName; String actualTblName = context.isTableNameEmpty() ? pks.get(0).getTable_name() : context.tableName; @@ -65,6 +59,7 @@ AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, pks, new ArrayList(), new ArrayList(), context.eventOnlyReplicationSpec()); Task addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf); + List> tasks = new ArrayList>(); tasks.add(addConstraintsTask); context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName); updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java index 0d9c700251..f0cb11ed75 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java @@ -49,11 +49,6 @@ } } - List> tasks = new ArrayList>(); - if (uks.isEmpty()) { - return tasks; - } - String actualDbName = context.isDbNameEmpty() ? uks.get(0).getTable_db() : context.dbName; String actualTblName = context.isTableNameEmpty() ? uks.get(0).getTable_name() : context.tableName; @@ -65,6 +60,7 @@ AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList(), new ArrayList(), uks, context.eventOnlyReplicationSpec()); Task addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf); + List> tasks = new ArrayList>(); tasks.add(addConstraintsTask); context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName); updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java index b99f5bc869..74c742c9e3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java @@ -117,13 +117,6 @@ public void setTableName(String tableName) { } /** - * @return the expectedType - */ - public TableType getExpectedType() { - return expectedType; - } - - /** * @return whether to expect a view being dropped */ public boolean getExpectView() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 8b64407d53..d7592bb966 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -627,11 +627,7 @@ synchronized private static void start(SessionState startSs, boolean isAsync, Lo try { if (startSs.tezSessionState == null) { - startSs.setTezSession(new TezSessionState(startSs.getSessionId(), startSs.sessionConf)); - } else { - // Only TezTask sets this, and then removes when done, so we don't expect to see it. - LOG.warn("Tez session was already present in SessionState before start: " - + startSs.tezSessionState); + startSs.setTezSession(new TezSessionState(startSs.getSessionId())); } if (startSs.tezSessionState.isOpen()) { return; @@ -644,9 +640,9 @@ synchronized private static void start(SessionState startSs, boolean isAsync, Lo } // Neither open nor opening. if (!isAsync) { - startSs.tezSessionState.open(); + startSs.tezSessionState.open(startSs.sessionConf); // should use conf on session start-up } else { - startSs.tezSessionState.beginOpen(null, console); + startSs.tezSessionState.beginOpen(startSs.sessionConf, null, console); } } catch (Exception e) { throw new RuntimeException(e); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java index 4e5d99134b..973c0cc630 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/SampleTezSessionState.java @@ -39,14 +39,13 @@ private boolean open; private final String sessionId; - private final HiveConf hiveConf; + private HiveConf hiveConf; private String user; private boolean doAsEnabled; - public SampleTezSessionState(String sessionId, TezSessionPoolManager parent, HiveConf conf) { - super(sessionId, parent, parent.getExpirationTracker(), conf); + public SampleTezSessionState(String sessionId, TezSessionPoolManager parent) { + super(sessionId, parent, parent.getExpirationTracker()); this.sessionId = sessionId; - this.hiveConf = conf; } @Override @@ -59,11 +58,12 @@ public void setOpen(boolean open) { } @Override - public void open() throws IOException, LoginException, URISyntaxException, + public void open(HiveConf conf) throws IOException, LoginException, URISyntaxException, TezException { + this.hiveConf = conf; UserGroupInformation ugi = Utils.getUGI(); user = ugi.getShortUserName(); - this.doAsEnabled = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS); + this.doAsEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS); setOpen(true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java index 5e1e68cfa8..d2b98c46ca 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java @@ -49,14 +49,8 @@ public TestTezSessionPoolManager() { } @Override - public void setupPool(HiveConf conf) throws InterruptedException { - conf.setVar(ConfVars.LLAP_TASK_SCHEDULER_AM_REGISTRY_NAME, ""); - super.setupPool(conf); - } - - @Override - public TezSessionPoolSession createSession(String sessionId, HiveConf conf) { - return new SampleTezSessionState(sessionId, this, conf); + public TezSessionPoolSession createSession(String sessionId) { + return new SampleTezSessionState(sessionId, this); } } @@ -194,7 +188,7 @@ public void testSessionReopen() { poolManager.reopenSession(session, conf); Mockito.verify(session).close(true); - Mockito.verify(session).open(new HashSet(), null); + Mockito.verify(session).open(conf, new HashSet(), null); // mocked session starts with default queue assertEquals("default", session.getQueueName()); @@ -331,7 +325,7 @@ public void testCloseAndOpenDefault() throws Exception { poolManager.reopenSession(session, conf); Mockito.verify(session).close(true); - Mockito.verify(session).open(new HashSet(), null); + Mockito.verify(session).open(conf, new HashSet(), null); } @Test diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java index 9b9eead0af..176692b6e5 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java @@ -227,7 +227,8 @@ public void testSubmit() throws Exception { task.submit(conf, dag, path, appLr, sessionState, Collections. emptyList(), new String[0], Collections. emptyMap()); // validate close/reopen - verify(sessionState, times(1)).open(any(Collection.class), any(Path.class)); + verify(sessionState, times(1)).open( + any(HiveConf.class), any(Collection.class), any(Path.class)); verify(sessionState, times(1)).close(eq(true)); // now uses pool after HIVE-7043 verify(session, times(2)).submitDAG(any(DAG.class)); } diff --git a/ql/src/test/queries/clientnegative/merge_negative_4.q b/ql/src/test/queries/clientnegative/merge_negative_4.q deleted file mode 100644 index c2b9254474..0000000000 --- a/ql/src/test/queries/clientnegative/merge_negative_4.q +++ /dev/null @@ -1,6 +0,0 @@ - - - -create external table srcpart2 (key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 2 buckets stored as RCFILE; -insert overwrite table srcpart2 partition (ds='2011') select * from src; -alter table srcpart2 partition (ds = '2011') concatenate; diff --git a/ql/src/test/queries/clientnegative/merge_negative_5.q b/ql/src/test/queries/clientnegative/merge_negative_5.q deleted file mode 100644 index 8039676a7b..0000000000 --- a/ql/src/test/queries/clientnegative/merge_negative_5.q +++ /dev/null @@ -1,14 +0,0 @@ -set hive.mapred.mode=nonstrict; -set hive.support.concurrency=true; -set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; -set hive.exec.dynamic.partition.mode=nonstrict; -set hive.vectorized.execution.enabled=false; -set hive.explain.user=false; -set hive.merge.cardinality.check=true; - -drop table if exists srcpart_acid; -CREATE TABLE srcpart_acid (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default'); -insert into srcpart_acid PARTITION (ds, hr) select * from srcpart; -insert into srcpart_acid PARTITION (ds, hr) select * from srcpart; - -alter table srcpart_acid partition(ds='2008-04-08',hr=='11') concatenate; diff --git a/ql/src/test/queries/clientpositive/mapjoin_hint.q b/ql/src/test/queries/clientpositive/mapjoin_hint.q deleted file mode 100644 index 75bebfa504..0000000000 --- a/ql/src/test/queries/clientpositive/mapjoin_hint.q +++ /dev/null @@ -1,69 +0,0 @@ -set hive.compute.query.using.stats=false; -set hive.mapred.mode=nonstrict; -set hive.explain.user=false; -set hive.optimize.ppd=true; -set hive.ppd.remove.duplicatefilters=true; -set hive.tez.dynamic.partition.pruning=true; -set hive.tez.dynamic.semijoin.reduction=true; -set hive.optimize.metadataonly=false; -set hive.optimize.index.filter=true; -set hive.stats.autogather=true; -set hive.tez.bigtable.minsize.semijoin.reduction=1; -set hive.tez.min.bloom.filter.entries=1; -set hive.stats.fetch.column.stats=true; - --- Create Tables -create table srcpart_date (key string, value string) partitioned by (ds string ) stored as ORC; -CREATE TABLE srcpart_small(key1 STRING, value1 STRING) partitioned by (ds string) STORED as ORC; - --- Add Partitions -alter table srcpart_date add partition (ds = "2008-04-08"); -alter table srcpart_date add partition (ds = "2008-04-09"); - -alter table srcpart_small add partition (ds = "2008-04-08"); -alter table srcpart_small add partition (ds = "2008-04-09"); - --- Load -insert overwrite table srcpart_date partition (ds = "2008-04-08" ) select key, value from srcpart where ds = "2008-04-08"; -insert overwrite table srcpart_date partition (ds = "2008-04-09") select key, value from srcpart where ds = "2008-04-09"; -insert overwrite table srcpart_small partition (ds = "2008-04-09") select key, value from srcpart where ds = "2008-04-09" limit 20; - - -analyze table srcpart_date compute statistics for columns; -analyze table srcpart_small compute statistics for columns; - -set hive.auto.convert.join=true; -set hive.auto.convert.join.noconditionaltask=true; -set hive.auto.convert.join.noconditionaltask.size=100000000000; - ---HIVE-17475 -EXPLAIN select /*+ mapjoin(None)*/ count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1); -EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1); - - --- Ensure that hint works even with CBO on, on a query with subquery. -create table tnull(i int, c char(2)); -insert into tnull values(NULL, NULL), (NULL, NULL); - -create table tempty(c char(2)); - -CREATE TABLE part_null( -p_partkey INT, -p_name STRING, -p_mfgr STRING, -p_brand STRING, -p_type STRING, -p_size INT, -p_container STRING, -p_retailprice DOUBLE, -p_comment STRING -) -ROW FORMAT DELIMITED FIELDS TERMINATED BY "," -; - -LOAD DATA LOCAL INPATH '../../data/files/part_tiny_nulls.txt' overwrite into table part_null; - -insert into part_null values(78487,NULL,'Manufacturer#6','Brand#52','LARGE BRUSHED BRASS', 23, 'MED BAG',1464.48,'hely blith'); - -explain select /*+ mapjoin(None)*/ * from part where p_name = (select p_name from part_null where p_name is null); -explain select * from part where p_name = (select p_name from part_null where p_name is null); diff --git a/ql/src/test/queries/clientpositive/orc_merge13.q b/ql/src/test/queries/clientpositive/orc_merge13.q deleted file mode 100644 index 8f013a0d09..0000000000 --- a/ql/src/test/queries/clientpositive/orc_merge13.q +++ /dev/null @@ -1,44 +0,0 @@ -drop table aa; -create table aa (a string, b int) stored as orc; -insert into table aa values("b",2); -insert into table aa values("c",3); - --- SORT_QUERY_RESULTS - -dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/aa/000000_0 ${hiveconf:hive.metastore.warehouse.dir}/aa/part-00000; -dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/aa/000000_0_copy_1 ${hiveconf:hive.metastore.warehouse.dir}/aa/part-00000_copy_1; - -select * from aa; - -alter table aa add columns(aa string, bb int); - -insert into table aa values("b",2,"b",2); -insert into table aa values("c",3,"c",3); - -dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/aa/000000_0 ${hiveconf:hive.metastore.warehouse.dir}/aa/part-00001; -dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/aa/000000_0_copy_1 ${hiveconf:hive.metastore.warehouse.dir}/aa/part-00001_copy_1; - -select * from aa; -select count(*) from aa; -select sum(hash(*)) from aa; - --- try concatenate multiple times (order of files chosen for concatenation is not guaranteed) -alter table aa concatenate; -select * from aa; -select count(*) from aa; -select sum(hash(*)) from aa; - -alter table aa concatenate; -select * from aa; -select count(*) from aa; -select sum(hash(*)) from aa; - -alter table aa concatenate; -select * from aa; -select count(*) from aa; -select sum(hash(*)) from aa; - -alter table aa concatenate; -select * from aa; -select count(*) from aa; -select sum(hash(*)) from aa; diff --git a/ql/src/test/results/clientnegative/merge_negative_3.q.out b/ql/src/test/results/clientnegative/merge_negative_3.q.out index 02c2ad19d0..906336d4d3 100644 --- a/ql/src/test/results/clientnegative/merge_negative_3.q.out +++ b/ql/src/test/results/clientnegative/merge_negative_3.q.out @@ -16,4 +16,4 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@srcpart2@ds=2011 POSTHOOK: Lineage: srcpart2 PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: srcpart2 PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Concatenate/Merge can not be performed on bucketed tables +FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Merge can not perform on bucketized partition/table. diff --git a/ql/src/test/results/clientnegative/merge_negative_4.q.out b/ql/src/test/results/clientnegative/merge_negative_4.q.out deleted file mode 100644 index 975422e2f8..0000000000 --- a/ql/src/test/results/clientnegative/merge_negative_4.q.out +++ /dev/null @@ -1,19 +0,0 @@ -PREHOOK: query: create external table srcpart2 (key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 2 buckets stored as RCFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@srcpart2 -POSTHOOK: query: create external table srcpart2 (key int, value string) partitioned by (ds string) clustered by (key) sorted by (key) into 2 buckets stored as RCFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@srcpart2 -PREHOOK: query: insert overwrite table srcpart2 partition (ds='2011') select * from src -PREHOOK: type: QUERY -PREHOOK: Input: default@src -PREHOOK: Output: default@srcpart2@ds=2011 -POSTHOOK: query: insert overwrite table srcpart2 partition (ds='2011') select * from src -POSTHOOK: type: QUERY -POSTHOOK: Input: default@src -POSTHOOK: Output: default@srcpart2@ds=2011 -POSTHOOK: Lineage: srcpart2 PARTITION(ds=2011).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart2 PARTITION(ds=2011).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Concatenate/Merge can not be performed on bucketed tables diff --git a/ql/src/test/results/clientnegative/merge_negative_5.q.out b/ql/src/test/results/clientnegative/merge_negative_5.q.out deleted file mode 100644 index 56a21a0236..0000000000 --- a/ql/src/test/results/clientnegative/merge_negative_5.q.out +++ /dev/null @@ -1,67 +0,0 @@ -PREHOOK: query: drop table if exists srcpart_acid -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table if exists srcpart_acid -POSTHOOK: type: DROPTABLE -PREHOOK: query: CREATE TABLE srcpart_acid (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@srcpart_acid -POSTHOOK: query: CREATE TABLE srcpart_acid (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='default') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@srcpart_acid -PREHOOK: query: insert into srcpart_acid PARTITION (ds, hr) select * from srcpart -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -PREHOOK: Output: default@srcpart_acid -POSTHOOK: query: insert into srcpart_acid PARTITION (ds, hr) select * from srcpart -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11 -POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12 -POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11 -POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12 -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert into srcpart_acid PARTITION (ds, hr) select * from srcpart -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -PREHOOK: Output: default@srcpart_acid -POSTHOOK: query: insert into srcpart_acid PARTITION (ds, hr) select * from srcpart -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=11 -POSTHOOK: Output: default@srcpart_acid@ds=2008-04-08/hr=12 -POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=11 -POSTHOOK: Output: default@srcpart_acid@ds=2008-04-09/hr=12 -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Concatenate/Merge can not be performed on transactional tables diff --git a/ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out b/ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out deleted file mode 100644 index 6a43f0d3dc..0000000000 --- a/ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out +++ /dev/null @@ -1,643 +0,0 @@ -PREHOOK: query: create table srcpart_date (key string, value string) partitioned by (ds string ) stored as ORC -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@srcpart_date -POSTHOOK: query: create table srcpart_date (key string, value string) partitioned by (ds string ) stored as ORC -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@srcpart_date -PREHOOK: query: CREATE TABLE srcpart_small(key1 STRING, value1 STRING) partitioned by (ds string) STORED as ORC -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@srcpart_small -POSTHOOK: query: CREATE TABLE srcpart_small(key1 STRING, value1 STRING) partitioned by (ds string) STORED as ORC -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@srcpart_small -PREHOOK: query: alter table srcpart_date add partition (ds = "2008-04-08") -PREHOOK: type: ALTERTABLE_ADDPARTS -PREHOOK: Output: default@srcpart_date -POSTHOOK: query: alter table srcpart_date add partition (ds = "2008-04-08") -POSTHOOK: type: ALTERTABLE_ADDPARTS -POSTHOOK: Output: default@srcpart_date -POSTHOOK: Output: default@srcpart_date@ds=2008-04-08 -PREHOOK: query: alter table srcpart_date add partition (ds = "2008-04-09") -PREHOOK: type: ALTERTABLE_ADDPARTS -PREHOOK: Output: default@srcpart_date -POSTHOOK: query: alter table srcpart_date add partition (ds = "2008-04-09") -POSTHOOK: type: ALTERTABLE_ADDPARTS -POSTHOOK: Output: default@srcpart_date -POSTHOOK: Output: default@srcpart_date@ds=2008-04-09 -PREHOOK: query: alter table srcpart_small add partition (ds = "2008-04-08") -PREHOOK: type: ALTERTABLE_ADDPARTS -PREHOOK: Output: default@srcpart_small -POSTHOOK: query: alter table srcpart_small add partition (ds = "2008-04-08") -POSTHOOK: type: ALTERTABLE_ADDPARTS -POSTHOOK: Output: default@srcpart_small -POSTHOOK: Output: default@srcpart_small@ds=2008-04-08 -PREHOOK: query: alter table srcpart_small add partition (ds = "2008-04-09") -PREHOOK: type: ALTERTABLE_ADDPARTS -PREHOOK: Output: default@srcpart_small -POSTHOOK: query: alter table srcpart_small add partition (ds = "2008-04-09") -POSTHOOK: type: ALTERTABLE_ADDPARTS -POSTHOOK: Output: default@srcpart_small -POSTHOOK: Output: default@srcpart_small@ds=2008-04-09 -PREHOOK: query: insert overwrite table srcpart_date partition (ds = "2008-04-08" ) select key, value from srcpart where ds = "2008-04-08" -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -PREHOOK: Output: default@srcpart_date@ds=2008-04-08 -POSTHOOK: query: insert overwrite table srcpart_date partition (ds = "2008-04-08" ) select key, value from srcpart where ds = "2008-04-08" -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 -POSTHOOK: Output: default@srcpart_date@ds=2008-04-08 -POSTHOOK: Lineage: srcpart_date PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_date PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table srcpart_date partition (ds = "2008-04-09") select key, value from srcpart where ds = "2008-04-09" -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -PREHOOK: Output: default@srcpart_date@ds=2008-04-09 -POSTHOOK: query: insert overwrite table srcpart_date partition (ds = "2008-04-09") select key, value from srcpart where ds = "2008-04-09" -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -POSTHOOK: Output: default@srcpart_date@ds=2008-04-09 -POSTHOOK: Lineage: srcpart_date PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_date PARTITION(ds=2008-04-09).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: insert overwrite table srcpart_small partition (ds = "2008-04-09") select key, value from srcpart where ds = "2008-04-09" limit 20 -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -PREHOOK: Output: default@srcpart_small@ds=2008-04-09 -POSTHOOK: query: insert overwrite table srcpart_small partition (ds = "2008-04-09") select key, value from srcpart where ds = "2008-04-09" limit 20 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 -POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 -POSTHOOK: Output: default@srcpart_small@ds=2008-04-09 -POSTHOOK: Lineage: srcpart_small PARTITION(ds=2008-04-09).key1 SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: srcpart_small PARTITION(ds=2008-04-09).value1 SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: analyze table srcpart_date compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart_date -PREHOOK: Input: default@srcpart_date@ds=2008-04-08 -PREHOOK: Input: default@srcpart_date@ds=2008-04-09 -PREHOOK: Output: default@srcpart_date -PREHOOK: Output: default@srcpart_date@ds=2008-04-08 -PREHOOK: Output: default@srcpart_date@ds=2008-04-09 -#### A masked pattern was here #### -POSTHOOK: query: analyze table srcpart_date compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart_date -POSTHOOK: Input: default@srcpart_date@ds=2008-04-08 -POSTHOOK: Input: default@srcpart_date@ds=2008-04-09 -POSTHOOK: Output: default@srcpart_date -POSTHOOK: Output: default@srcpart_date@ds=2008-04-08 -POSTHOOK: Output: default@srcpart_date@ds=2008-04-09 -#### A masked pattern was here #### -PREHOOK: query: analyze table srcpart_small compute statistics for columns -PREHOOK: type: QUERY -PREHOOK: Input: default@srcpart_small -PREHOOK: Input: default@srcpart_small@ds=2008-04-08 -PREHOOK: Input: default@srcpart_small@ds=2008-04-09 -PREHOOK: Output: default@srcpart_small -PREHOOK: Output: default@srcpart_small@ds=2008-04-08 -PREHOOK: Output: default@srcpart_small@ds=2008-04-09 -#### A masked pattern was here #### -POSTHOOK: query: analyze table srcpart_small compute statistics for columns -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcpart_small -POSTHOOK: Input: default@srcpart_small@ds=2008-04-08 -POSTHOOK: Input: default@srcpart_small@ds=2008-04-09 -POSTHOOK: Output: default@srcpart_small -POSTHOOK: Output: default@srcpart_small@ds=2008-04-08 -POSTHOOK: Output: default@srcpart_small@ds=2008-04-09 -#### A masked pattern was here #### -PREHOOK: query: EXPLAIN select /*+ mapjoin(None)*/ count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN select /*+ mapjoin(None)*/ count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Reducer 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) - Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key1_min) AND DynamicValue(RS_7_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key1_bloom_filter)))) (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key1_min) AND DynamicValue(RS_7_srcpart_small_key1_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key1_bloom_filter)))) (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Execution mode: llap - LLAP IO: all inputs - Map 4 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: _col0 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=40) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - Statistics: Num rows: 129 Data size: 1032 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: bigint) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 5 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=40) - mode: final - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary) - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) -PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN select count(*) from srcpart_date join srcpart_small on (srcpart_date.key = srcpart_small.key1) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Map 1 <- Map 3 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: srcpart_date - filterExpr: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: string) - 1 _col0 (type: string) - input vertices: - 1 Map 3 - Statistics: Num rows: 129 Data size: 1032 Basic stats: COMPLETE Column stats: PARTIAL - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: all inputs - Map 3 - Map Operator Tree: - TableScan - alias: srcpart_small - filterExpr: key1 is not null (type: boolean) - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL - Filter Operator - predicate: key1 is not null (type: boolean) - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: key1 (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL - Execution mode: llap - LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: create table tnull(i int, c char(2)) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tnull -POSTHOOK: query: create table tnull(i int, c char(2)) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tnull -PREHOOK: query: insert into tnull values(NULL, NULL), (NULL, NULL) -PREHOOK: type: QUERY -PREHOOK: Output: default@tnull -POSTHOOK: query: insert into tnull values(NULL, NULL), (NULL, NULL) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@tnull -POSTHOOK: Lineage: tnull.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -POSTHOOK: Lineage: tnull.i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -PREHOOK: query: create table tempty(c char(2)) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@tempty -POSTHOOK: query: create table tempty(c char(2)) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@tempty -PREHOOK: query: CREATE TABLE part_null( -p_partkey INT, -p_name STRING, -p_mfgr STRING, -p_brand STRING, -p_type STRING, -p_size INT, -p_container STRING, -p_retailprice DOUBLE, -p_comment STRING -) -ROW FORMAT DELIMITED FIELDS TERMINATED BY "," -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@part_null -POSTHOOK: query: CREATE TABLE part_null( -p_partkey INT, -p_name STRING, -p_mfgr STRING, -p_brand STRING, -p_type STRING, -p_size INT, -p_container STRING, -p_retailprice DOUBLE, -p_comment STRING -) -ROW FORMAT DELIMITED FIELDS TERMINATED BY "," -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@part_null -PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny_nulls.txt' overwrite into table part_null -PREHOOK: type: LOAD -#### A masked pattern was here #### -PREHOOK: Output: default@part_null -POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny_nulls.txt' overwrite into table part_null -POSTHOOK: type: LOAD -#### A masked pattern was here #### -POSTHOOK: Output: default@part_null -PREHOOK: query: insert into part_null values(78487,NULL,'Manufacturer#6','Brand#52','LARGE BRUSHED BRASS', 23, 'MED BAG',1464.48,'hely blith') -PREHOOK: type: QUERY -PREHOOK: Output: default@part_null -POSTHOOK: query: insert into part_null values(78487,NULL,'Manufacturer#6','Brand#52','LARGE BRUSHED BRASS', 23, 'MED BAG',1464.48,'hely blith') -POSTHOOK: type: QUERY -POSTHOOK: Output: default@part_null -POSTHOOK: Lineage: part_null.p_brand SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col4, type:string, comment:), ] -POSTHOOK: Lineage: part_null.p_comment SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col9, type:string, comment:), ] -POSTHOOK: Lineage: part_null.p_container SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col7, type:string, comment:), ] -POSTHOOK: Lineage: part_null.p_mfgr SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] -POSTHOOK: Lineage: part_null.p_name SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -POSTHOOK: Lineage: part_null.p_partkey EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: part_null.p_retailprice EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col8, type:string, comment:), ] -POSTHOOK: Lineage: part_null.p_size EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col6, type:string, comment:), ] -POSTHOOK: Lineage: part_null.p_type SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col5, type:string, comment:), ] -Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product -PREHOOK: query: explain select /*+ mapjoin(None)*/ * from part where p_name = (select p_name from part_null where p_name is null) -PREHOOK: type: QUERY -POSTHOOK: query: explain select /*+ mapjoin(None)*/ * from part where p_name = (select p_name from part_null where p_name is null) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: part - filterExpr: (p_name = null) (type: boolean) - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (p_name = null) (type: boolean) - Statistics: Num rows: 1 Data size: 619 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 3 - Map Operator Tree: - TableScan - alias: part_null - filterExpr: p_name is null (type: boolean) - Statistics: Num rows: 32 Data size: 5888 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Merge Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 - 1 - 2 - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), null (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 3' is a cross product -PREHOOK: query: explain select * from part where p_name = (select p_name from part_null where p_name is null) -PREHOOK: type: QUERY -POSTHOOK: query: explain select * from part where p_name = (select p_name from part_null where p_name is null) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (CUSTOM_SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: part - filterExpr: (p_name = null) (type: boolean) - Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE - Filter Operator - predicate: (p_name = null) (type: boolean) - Statistics: Num rows: 1 Data size: 619 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - Execution mode: llap - LLAP IO: no inputs - Map 2 - Map Operator Tree: - TableScan - alias: part_null - filterExpr: p_name is null (type: boolean) - Statistics: Num rows: 32 Data size: 5888 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Execution mode: llap - LLAP IO: no inputs - Map 4 - Map Operator Tree: - TableScan - alias: part_null - filterExpr: p_name is null (type: boolean) - Statistics: Num rows: 32 Data size: 5888 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: p_name is null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (sq_count_check(_col0) <= 1) (type: boolean) - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 - 1 - 2 - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - input vertices: - 0 Map 1 - 2 Map 4 - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), null (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - diff --git a/ql/src/test/results/clientpositive/orc_merge13.q.out b/ql/src/test/results/clientpositive/orc_merge13.q.out deleted file mode 100644 index 70ae731f29..0000000000 --- a/ql/src/test/results/clientpositive/orc_merge13.q.out +++ /dev/null @@ -1,248 +0,0 @@ -PREHOOK: query: drop table aa -PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table aa -POSTHOOK: type: DROPTABLE -PREHOOK: query: create table aa (a string, b int) stored as orc -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@aa -POSTHOOK: query: create table aa (a string, b int) stored as orc -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@aa -PREHOOK: query: insert into table aa values("b",2) -PREHOOK: type: QUERY -PREHOOK: Output: default@aa -POSTHOOK: query: insert into table aa values("b",2) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@aa -POSTHOOK: Lineage: aa.a SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: aa.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -PREHOOK: query: insert into table aa values("c",3) -PREHOOK: type: QUERY -PREHOOK: Output: default@aa -POSTHOOK: query: insert into table aa values("c",3) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@aa -POSTHOOK: Lineage: aa.a SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: aa.b EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -PREHOOK: query: select * from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select * from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -b 2 -c 3 -PREHOOK: query: alter table aa add columns(aa string, bb int) -PREHOOK: type: ALTERTABLE_ADDCOLS -PREHOOK: Input: default@aa -PREHOOK: Output: default@aa -POSTHOOK: query: alter table aa add columns(aa string, bb int) -POSTHOOK: type: ALTERTABLE_ADDCOLS -POSTHOOK: Input: default@aa -POSTHOOK: Output: default@aa -PREHOOK: query: insert into table aa values("b",2,"b",2) -PREHOOK: type: QUERY -PREHOOK: Output: default@aa -POSTHOOK: query: insert into table aa values("b",2,"b",2) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@aa -POSTHOOK: Lineage: aa.a SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: aa.aa SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] -POSTHOOK: Lineage: aa.b EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -POSTHOOK: Lineage: aa.bb EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col4, type:string, comment:), ] -PREHOOK: query: insert into table aa values("c",3,"c",3) -PREHOOK: type: QUERY -PREHOOK: Output: default@aa -POSTHOOK: query: insert into table aa values("c",3,"c",3) -POSTHOOK: type: QUERY -POSTHOOK: Output: default@aa -POSTHOOK: Lineage: aa.a SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] -POSTHOOK: Lineage: aa.aa SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col3, type:string, comment:), ] -POSTHOOK: Lineage: aa.b EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] -POSTHOOK: Lineage: aa.bb EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col4, type:string, comment:), ] -PREHOOK: query: select * from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select * from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -b 2 NULL NULL -b 2 b 2 -c 3 NULL NULL -c 3 c 3 -PREHOOK: query: select count(*) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -4 -PREHOOK: query: select sum(hash(*)) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select sum(hash(*)) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -11753376 -PREHOOK: query: alter table aa concatenate -PREHOOK: type: ALTER_TABLE_MERGE -PREHOOK: Input: default@aa -PREHOOK: Output: default@aa -POSTHOOK: query: alter table aa concatenate -POSTHOOK: type: ALTER_TABLE_MERGE -POSTHOOK: Input: default@aa -POSTHOOK: Output: default@aa -PREHOOK: query: select * from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select * from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -b 2 NULL NULL -b 2 b 2 -c 3 NULL NULL -c 3 c 3 -PREHOOK: query: select count(*) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -4 -PREHOOK: query: select sum(hash(*)) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select sum(hash(*)) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -11753376 -PREHOOK: query: alter table aa concatenate -PREHOOK: type: ALTER_TABLE_MERGE -PREHOOK: Input: default@aa -PREHOOK: Output: default@aa -POSTHOOK: query: alter table aa concatenate -POSTHOOK: type: ALTER_TABLE_MERGE -POSTHOOK: Input: default@aa -POSTHOOK: Output: default@aa -PREHOOK: query: select * from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select * from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -b 2 NULL NULL -b 2 b 2 -c 3 NULL NULL -c 3 c 3 -PREHOOK: query: select count(*) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -4 -PREHOOK: query: select sum(hash(*)) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select sum(hash(*)) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -11753376 -PREHOOK: query: alter table aa concatenate -PREHOOK: type: ALTER_TABLE_MERGE -PREHOOK: Input: default@aa -PREHOOK: Output: default@aa -POSTHOOK: query: alter table aa concatenate -POSTHOOK: type: ALTER_TABLE_MERGE -POSTHOOK: Input: default@aa -POSTHOOK: Output: default@aa -PREHOOK: query: select * from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select * from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -b 2 NULL NULL -b 2 b 2 -c 3 NULL NULL -c 3 c 3 -PREHOOK: query: select count(*) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -4 -PREHOOK: query: select sum(hash(*)) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select sum(hash(*)) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -11753376 -PREHOOK: query: alter table aa concatenate -PREHOOK: type: ALTER_TABLE_MERGE -PREHOOK: Input: default@aa -PREHOOK: Output: default@aa -POSTHOOK: query: alter table aa concatenate -POSTHOOK: type: ALTER_TABLE_MERGE -POSTHOOK: Input: default@aa -POSTHOOK: Output: default@aa -PREHOOK: query: select * from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select * from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -b 2 NULL NULL -b 2 b 2 -c 3 NULL NULL -c 3 c 3 -PREHOOK: query: select count(*) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select count(*) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -4 -PREHOOK: query: select sum(hash(*)) from aa -PREHOOK: type: QUERY -PREHOOK: Input: default@aa -#### A masked pattern was here #### -POSTHOOK: query: select sum(hash(*)) from aa -POSTHOOK: type: QUERY -POSTHOOK: Input: default@aa -#### A masked pattern was here #### -11753376 diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java index 3790d3cb33..ef77daf221 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java @@ -224,13 +224,15 @@ public void writeDouble(double v) throws IOException { /* * STRING. - * + * * Can be used to write CHAR and VARCHAR when the caller takes responsibility for * truncation/padding issues. */ @Override public void writeString(byte[] v) throws IOException { beginPrimitive(); + if (v.equals(nullSequenceBytes)) { + } LazyUtils.writeEscaped(output, v, 0, v.length, isEscaped, escapeChar, needsEscape); finishPrimitive();