From 4ac488902e690bcfadd5fd9ac95351447f4e7230 Mon Sep 17 00:00:00 2001 From: stack Date: Wed, 25 Nov 2015 16:25:43 -0800 Subject: [PATCH] Add in a ClusterSchema Interface. It will have all API for all cluster manipulation; adding namespaces, tables, amending column families, etc. The idea is to gather up our mess and put it all behind a tidy API that all works the same way returning a Future to wait on. This patch does namespace operations first. M hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java Just remove a hyphen from thread name. We were doubling up. A hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java A very basic 'Service' Interface. To be enhanced later or replaced by the guava Service (our guava is too old currently and its Service is not as ) M hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java Moved this back to hbase-common so coul use it in new Service Interface. A hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java New ClusterSchema Interface. Has namespace API only and some utility as well as a low-level API to fetch namespace table manager that probably belongs elsewhere. A hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java A ClusterSchemeException. I don't think this is really needed so will probably remove in later patch. A hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java Interface that is made of ClusterSchema and Service A hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java Implementation. Moved stuff here from Master. M hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Mix in ClusterSchema for namespace ops. All that remains in here is pre and post coprocessor calls. Some cleanup around namespace init; make it less complicated. Namespace ops don't have to be public anymore. M hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java Use new ClusterSchema when doing namespace ops. M hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java Remove namespace particular ops and add getting of a ClusterSchema instance instead. A hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java Map procedure processing to a Future model. Probably belongs over in procedure space. Can move later if generally useful. M hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java Cleanup M hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java M hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java M hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java --- .../hbase/client/ConnectionImplementation.java | 2 +- .../main/java/org/apache/hadoop/hbase/Service.java | 39 +++ .../hadoop/hbase/ServiceNotRunningException.java | 38 +++ .../org/apache/hadoop/hbase/ResourceChecker.java | 2 +- .../apache/hadoop/hbase/master/ClusterSchema.java | 135 ++++++++++ .../hbase/master/ClusterSchemaException.java | 41 +++ .../hadoop/hbase/master/ClusterSchemaService.java | 27 ++ .../hbase/master/ClusterSchemaServiceImpl.java | 181 ++++++++++++++ .../org/apache/hadoop/hbase/master/HMaster.java | 274 ++++++++------------- .../hadoop/hbase/master/MasterRpcServices.java | 13 +- .../apache/hadoop/hbase/master/MasterServices.java | 78 +----- .../hadoop/hbase/master/ProcedureFuture.java | 127 ++++++++++ .../hadoop/hbase/master/TableNamespaceManager.java | 67 ++--- .../master/procedure/CreateNamespaceProcedure.java | 2 +- .../master/procedure/DeleteNamespaceProcedure.java | 2 +- .../master/procedure/ModifyNamespaceProcedure.java | 3 +- .../hbase/namespace/NamespaceStateManager.java | 6 +- .../org/apache/hadoop/hbase/TestNamespace.java | 1 - .../hadoop/hbase/master/TestCatalogJanitor.java | 59 +---- .../hadoop/hbase/master/TestMasterNoCluster.java | 8 - 20 files changed, 757 insertions(+), 348 deletions(-) create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 62a7998..cb4bfe0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -395,7 +395,7 @@ class ConnectionImplementation implements ClusterConnection, Closeable { synchronized (this) { if (batchPool == null) { this.batchPool = getThreadPool(conf.getInt("hbase.hconnection.threads.max", 256), - conf.getInt("hbase.hconnection.threads.core", 256), "-shared-", null); + conf.getInt("hbase.hconnection.threads.core", 256), "-shared", null); this.cleanupPool = true; } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java new file mode 100644 index 0000000..3afa4cd --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * Simple Service. + */ +// TODO: Move on to guava Service after we update our guava version; later guava has nicer +// Service implmentation. +@InterfaceAudience.Private +public interface Service { + /** + * State copied from guava. Kept internal for now till more sophisticated need than what is + * provided below. + */ + enum State {NEW, STARTING, RUNNING, STOPPING, TERMINATED}; + + void startAndWait() throws IOException; + boolean isRunning(); + void stopAndWait() throws IOException; +} \ No newline at end of file diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java new file mode 100644 index 0000000..e1b5773 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java @@ -0,0 +1,38 @@ +package org.apache.hadoop.hbase; +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +@SuppressWarnings("serial") +@InterfaceAudience.Private +public class ServiceNotRunningException extends HBaseIOException { + public ServiceNotRunningException() { + } + + public ServiceNotRunningException(String message) { + super(message); + } + + public ServiceNotRunningException(String message, Throwable cause) { + super(message, cause); + } + + public ServiceNotRunningException(Throwable cause) { + super(cause); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java index 539aea3..c3f699d 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java @@ -41,7 +41,7 @@ public class ResourceChecker { /** * Constructor - * @param tagLine - the tagLine is added to the logs. Must be be null. + * @param tagLine - the tagLine is added to the logs. Must not be be null. */ public ResourceChecker(final String tagLine) { this.tagLine = tagLine; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java new file mode 100644 index 0000000..e4b17ef --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.List; +import java.util.concurrent.Future; + +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.ProcedureInfo; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * View and edit the current cluster schema. + * + *

Implementation Notes

+ * Nonces are for when operation is non-idempotent to ensure once-only semantic, even + * across process failures. + */ +// ClusterSchema is private to the Master; only the Master knows current cluster state and has +// means of editing/altering it. +// TODO: Remove Server when MasterServices +@InterfaceAudience.Private +public interface ClusterSchema { + /** + * Timeout for cluster operations in milliseconds. + */ + public static final String HBASE_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY = + "hbase.cluster.schema.operation.timeout"; + /** + * Default operation timeout in milliseconds. + */ + public static final int DEFAULT_HBASE_CLUSTER_SCHEMA_OPERATION_TIMEOUT = 5 * 60 * 1000; + + /** + * Utility method that will wait {@link #HBASE_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY} + * milliseconds on the passed in future and if exceptions, throws any found (after + * conversion): i.e. {@link InterruptedException} becomes {@link InterruptedIOException} or + * ProcedureInfo.getForeignExceptionMessage() becomes the hosted NamespaceNotFoundException, and + * so on. + * + * <> + * + * @param future Future to wait on. + * @return On completion, info on the procedure that ran. + * @throws IOException Exception waiting on Future or the exception that came back in the + * ProcedureInfo. + */ + ProcedureInfo await(final Future future) throws IOException; + + /** + * For internals use only. Do not use. Provisionally part of this Interface. + * Prefer the high-level APIs available elsewhere in this API. + * @return Instance of {@link TableNamespaceManager} + */ + // TODO: Remove from here. Keep internal. This Interface is too high-level to host this accessor. + TableNamespaceManager getTableNamespaceManager(); + + /** + * Create a new Namespace. + * @param namespaceDescriptor descriptor for new Namespace + * @param nonceGroup Identifier for the source of the request, a client or process. + * @param nonce A unique identifier for this operation from the client or process identified by + * nonceGroup (the source must ensure each operation gets a unique id). + * @return Operation Future. + * Use {@link Future#get(long, java.util.concurrent.TimeUnit)} to wait on completion. + * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException} + * as well as {@link IOException} + */ + Future createNamespace(NamespaceDescriptor namespaceDescriptor, long nonceGroup, + long nonce) + throws IOException; + + /** + * Modify an existing Namespace. + * @param nonceGroup Identifier for the source of the request, a client or process. + * @param nonce A unique identifier for this operation from the client or process identified by + * nonceGroup (the source must ensure each operation gets a unique id). + * @return Operation Future. + * Use {@link Future#get(long, java.util.concurrent.TimeUnit)} to wait on completion. + * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException} + * as well as {@link IOException} + */ + Future modifyNamespace(NamespaceDescriptor descriptor, long nonceGroup, + long nonce) + throws IOException; + + /** + * Delete an existing Namespace. + * Only empty Namespaces (no tables) can be removed. + * @param nonceGroup Identifier for the source of the request, a client or process. + * @param nonce A unique identifier for this operation from the client or process identified by + * nonceGroup (the source must ensure each operation gets a unique id). + * @return Operation Future. + * Use {@link Future#get(long, java.util.concurrent.TimeUnit)} to wait on completion. + * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException} + * as well as {@link IOException} + */ + Future deleteNamespace(String name, long nonceGroup, long nonce) + throws IOException; + + /** + * Get a Namespace + * @param name Name of the Namespace + * @return Namespace descriptor for name + * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException} + * as well as {@link IOException} + */ + // No Future here because presumption is that the request will go against cached metadata so + // return immediately -- no need of running a Procedure. + NamespaceDescriptor getNamespace(String name) throws IOException; + + /** + * Get all Namespaces + * @return All Namespace descriptors + * @throws IOException + */ + List getNamespaces() throws IOException; +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java new file mode 100644 index 0000000..9a1c187 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +@SuppressWarnings("serial") +@InterfaceAudience.Private +public class ClusterSchemaException extends HBaseIOException { + public ClusterSchemaException() { + // TODO Auto-generated constructor stub + } + + public ClusterSchemaException(String message) { + super(message); + } + + public ClusterSchemaException(String message, Throwable cause) { + super(message, cause); + } + + public ClusterSchemaException(Throwable cause) { + super(cause); + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java new file mode 100644 index 0000000..43353ba --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import org.apache.hadoop.hbase.Service; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * Mixes in ClusterSchema and Service + */ +@InterfaceAudience.Private +public interface ClusterSchemaService extends ClusterSchema, Service {} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java new file mode 100644 index 0000000..f548bff --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.NamespaceNotFoundException; +import org.apache.hadoop.hbase.ProcedureInfo; +import org.apache.hadoop.hbase.ServiceNotRunningException; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.exceptions.TimeoutIOException; +import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure; +import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.ModifyNamespaceProcedure; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.RemoteProcedureException; + +@InterfaceAudience.Private +class ClusterSchemaServiceImpl implements ClusterSchemaService { + // State is volatile. Only changed inside synchronized block. This is like double-checked locking. + // Double-checked locking is supposed to work ever since jdk5 so this approach should be ok. Have + // to do this for startup case. We create namespaces and tables as part of the startAndWait + // internally which makes calls back in here to do actual namespace and table creation. If all + // was synchronized, the thread trying to do the create (not 'this' thread but a thread out of + // an executor) would block. You can see the problem when you run TestProcedureManager when the + // startAndWait below calls start on TableNamespaceManager and it then goes and blocks creating + // system namespaces and table. + private volatile State serviceState = State.NEW; + private final int clusterSchemaOperationTimeoutInMillis; + private final TableNamespaceManager tableNamespaceManager; + private final MasterServices masterServices; + private final static List EMPTY_NAMESPACE_LIST = + Collections.unmodifiableList(new ArrayList(0)); + + ClusterSchemaServiceImpl(final MasterServices masterServices) { + this.masterServices = masterServices; + this.tableNamespaceManager = new TableNamespaceManager(masterServices); + this.clusterSchemaOperationTimeoutInMillis = this.masterServices.getConfiguration(). + getInt(ClusterSchema.HBASE_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY, + ClusterSchema.DEFAULT_HBASE_CLUSTER_SCHEMA_OPERATION_TIMEOUT); + } + + @Override + public boolean isRunning() { + return this.serviceState.equals(State.RUNNING); + } + + private void checkIsRunning() throws ServiceNotRunningException { + if (!isRunning()) throw new ServiceNotRunningException(); + } + + @Override + public synchronized void startAndWait() throws IOException { + if (!this.serviceState.equals(State.NEW)) { + throw new IllegalStateException(this.serviceState.toString()); + } + this.serviceState = State.STARTING; + this.tableNamespaceManager.start(); + this.serviceState = State.RUNNING; + } + + @Override + public synchronized void stopAndWait() throws IOException { + checkIsRunning(); + // You can't stop tableNamespaceManager. + this.serviceState = State.TERMINATED; + } + + @Override + public TableNamespaceManager getTableNamespaceManager() { + return this.tableNamespaceManager; + } + + private Future submitProcedure(final Procedure procedure, long nonceGroup, + long nonce) + throws ServiceNotRunningException { + // Allow submission of services if we are STARTING up or we are RUNNING. Fail for all us. + if (this.serviceState != State.RUNNING && this.serviceState != State.STARTING) { + throw new IllegalStateException(this.serviceState.toString()); + } + ProcedureExecutor pe = this.masterServices.getMasterProcedureExecutor(); + long procId = pe.submitProcedure(procedure, nonceGroup, nonce); + return new ProcedureFuture(pe, procId); + } + + @Override + public ProcedureInfo await(final Future future) + throws IOException { + try { + ProcedureInfo pi = + future.get(this.clusterSchemaOperationTimeoutInMillis, TimeUnit.MILLISECONDS); + // If the procedure fails, we should always have an exception captured. Throw it. + if (pi.isFailed()) { + throw RemoteProcedureException.fromProto(pi.getForeignExceptionMessage()). + unwrapRemoteException(); + } + return pi; + } catch (ExecutionException ee) { + // No cleanup to do... just let the exception out. + if (ee.getCause() instanceof IOException) throw (IOException)ee.getCause(); + else throw new ClusterSchemaException(ee.getCause()); + } catch (InterruptedException e) { + IOException ioe = new InterruptedIOException(); + ioe.initCause(e); + throw ioe; + } catch (TimeoutException e) { + throw new TimeoutIOException(future.toString()); + } + } + + @Override + public Future createNamespace(NamespaceDescriptor namespaceDescriptor, + long nonceGroup, long nonce) + throws IOException { + return submitProcedure(new CreateNamespaceProcedure( + this.masterServices.getMasterProcedureExecutor().getEnvironment(), namespaceDescriptor), + nonceGroup, nonce); + } + + @Override + public Future modifyNamespace(NamespaceDescriptor namespaceDescriptor, + long nonceGroup, long nonce) + throws IOException { + return submitProcedure(new ModifyNamespaceProcedure( + this.masterServices.getMasterProcedureExecutor().getEnvironment(), namespaceDescriptor), + nonceGroup, nonce); + } + + @Override + public Future deleteNamespace(String name, long nonceGroup, long nonce) + throws IOException { + return submitProcedure(new DeleteNamespaceProcedure( + this.masterServices.getMasterProcedureExecutor().getEnvironment(), name), + nonceGroup, nonce); + } + + @Override + public NamespaceDescriptor getNamespace(String name) throws IOException { + checkIsRunning(); + NamespaceDescriptor nsd = this.tableNamespaceManager.get(name); + if (nsd == null) throw new NamespaceNotFoundException(name); + return nsd; + } + + @Override + public List getNamespaces() throws IOException { + checkIsRunning(); + Set set = this.tableNamespaceManager.list(); + if (set == null || set.isEmpty()) return EMPTY_NAMESPACE_LIST; + List list = new ArrayList(set.size()); + list.addAll(set); + return Collections.unmodifiableList(list); + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 46c7f79..c34ae13 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -96,17 +96,14 @@ import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore; import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory; import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure; -import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure; import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure; import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure; -import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure; import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure; import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure; import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure; -import org.apache.hadoop.hbase.master.procedure.ModifyNamespaceProcedure; import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure; import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; @@ -184,7 +181,7 @@ import com.google.protobuf.Service; */ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) @SuppressWarnings("deprecation") -public class HMaster extends HRegionServer implements MasterServices, Server { +public class HMaster extends HRegionServer implements MasterServices { private static final Log LOG = LogFactory.getLog(HMaster.class.getName()); /** @@ -255,8 +252,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // Tracker for region normalizer state private RegionNormalizerTracker regionNormalizerTracker; - /** Namespace stuff */ - private TableNamespaceManager tableNamespaceManager; + private ClusterSchemaService clusterSchemaService; // Metrics for the HMaster final MetricsMaster metricsMaster; @@ -780,8 +776,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server { this.catalogJanitorChore = new CatalogJanitor(this, this); getChoreService().scheduleChore(catalogJanitorChore); - status.setStatus("Starting namespace manager"); - initNamespace(); + status.setStatus("Starting cluster schema service"); + initClusterSchemaService(); if (this.cpHost != null) { try { @@ -964,10 +960,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { status.setStatus("META assigned."); } - void initNamespace() throws IOException { - //create namespace manager - tableNamespaceManager = new TableNamespaceManager(this); - tableNamespaceManager.start(); + void initClusterSchemaService() throws IOException, InterruptedException { + this.clusterSchemaService = new ClusterSchemaServiceImpl(this); + this.clusterSchemaService.startAndWait(); + if (!this.clusterSchemaService.isRunning()) throw new HBaseIOException("Failed start"); } void initQuotaManager() throws IOException { @@ -1049,11 +1045,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return tableStateManager; } - @Override - public TableNamespaceManager getTableNamespaceManager() { - return tableNamespaceManager; - } - /* * Start up all services. If any of these threads gets an unhandled exception * then they just die with a logged message. This should be fine because @@ -1476,9 +1467,9 @@ public class HMaster extends HRegionServer implements MasterServices, Server { if (isStopped()) { throw new MasterNotRunningException(); } - + checkInitialized(); String namespace = hTableDescriptor.getTableName().getNamespaceAsString(); - ensureNamespaceExists(namespace); + this.clusterSchemaService.getNamespace(namespace); HRegionInfo[] newRegions = ModifyRegionUtils.createHRegionInfos(hTableDescriptor, splitKeys); checkInitialized(); @@ -2299,18 +2290,9 @@ public class HMaster extends HRegionServer implements MasterServices, Server { void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException { checkServiceStarted(); - if (!this.initialized) { - throw new PleaseHoldException("Master is initializing"); - } + if (!isInitialized()) throw new PleaseHoldException("Master is initializing"); } - void checkNamespaceManagerReady() throws IOException { - checkInitialized(); - if (tableNamespaceManager == null || - !tableNamespaceManager.isTableAvailableAndInitialized(true)) { - throw new IOException("Table Namespace Manager not ready yet, try again later"); - } - } /** * Report whether this master is currently the active master or not. * If not active master, we are parked on ZK waiting to become active. @@ -2446,138 +2428,112 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } @Override - public void createNamespace( - final NamespaceDescriptor descriptor, - final long nonceGroup, - final long nonce) throws IOException { - TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName())); - checkNamespaceManagerReady(); - if (cpHost != null) { - if (cpHost.preCreateNamespace(descriptor)) { - return; - } - } - createNamespaceSync(descriptor, nonceGroup, nonce); - if (cpHost != null) { - cpHost.postCreateNamespace(descriptor); - } + public ClusterSchema getClusterSchema() { + return this.clusterSchemaService; } - @Override - public void createNamespaceSync( - final NamespaceDescriptor descriptor, - final long nonceGroup, - final long nonce) throws IOException { - LOG.info(getClientIdAuditPrefix() + " creating " + descriptor); + /** + * Create a new Namespace. + * @param namespaceDescriptor descriptor for new Namespace + * @param nonceGroup Identifier for the source of the request, a client or process. + * @param nonce A unique identifier for this operation from the client or process identified by + * nonceGroup (the source must ensure each operation gets a unique id). + * @throws IOException + */ + void createNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup, + final long nonce) + throws IOException { + checkInitialized(); + TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName())); + if (this.cpHost != null && this.cpHost.preCreateNamespace(namespaceDescriptor)) return; + LOG.info(getClientIdAuditPrefix() + " creating " + namespaceDescriptor); // Execute the operation synchronously - wait for the operation to complete before continuing. - long procId = this.procedureExecutor.submitProcedure( - new CreateNamespaceProcedure(procedureExecutor.getEnvironment(), descriptor), - nonceGroup, - nonce); - ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId); + ClusterSchema cs = getClusterSchema(); + cs.await(cs.createNamespace(namespaceDescriptor, nonceGroup, nonce)); + if (this.cpHost != null) this.cpHost.postCreateNamespace(namespaceDescriptor); } - @Override - public void modifyNamespace( - final NamespaceDescriptor descriptor, - final long nonceGroup, - final long nonce) throws IOException { - TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName())); - checkNamespaceManagerReady(); - if (cpHost != null) { - if (cpHost.preModifyNamespace(descriptor)) { - return; - } - } - LOG.info(getClientIdAuditPrefix() + " modify " + descriptor); + /** + * Modify an existing Namespace. + * @param nonceGroup Identifier for the source of the request, a client or process. + * @param nonce A unique identifier for this operation from the client or process identified by + * nonceGroup (the source must ensure each operation gets a unique id).\ + * @throws IOException + */ + void modifyNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup, + final long nonce) + throws IOException { + checkInitialized(); + TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName())); + if (this.cpHost != null && this.cpHost.preModifyNamespace(namespaceDescriptor)) return; + LOG.info(getClientIdAuditPrefix() + " modify " + namespaceDescriptor); // Execute the operation synchronously - wait for the operation to complete before continuing. - long procId = this.procedureExecutor.submitProcedure( - new ModifyNamespaceProcedure(procedureExecutor.getEnvironment(), descriptor), - nonceGroup, - nonce); - ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId); - if (cpHost != null) { - cpHost.postModifyNamespace(descriptor); - } + ClusterSchema cs = getClusterSchema(); + cs.await(cs.modifyNamespace(namespaceDescriptor, nonceGroup, nonce)); + if (this.cpHost != null) this.cpHost.postModifyNamespace(namespaceDescriptor); } - @Override - public void deleteNamespace( - final String name, - final long nonceGroup, - final long nonce) throws IOException { - checkNamespaceManagerReady(); - if (cpHost != null) { - if (cpHost.preDeleteNamespace(name)) { - return; - } - } + /** + * Delete an existing Namespace. Only empty Namespaces (no tables) can be removed. + * @param nonceGroup Identifier for the source of the request, a client or process. + * @param nonce A unique identifier for this operation from the client or process identified by + * nonceGroup (the source must ensure each operation gets a unique id). + * @throws IOException + */ + void deleteNamespace(final String name, final long nonceGroup, final long nonce) + throws IOException { + checkInitialized(); + if (this.cpHost != null && this.cpHost.preDeleteNamespace(name)) return; LOG.info(getClientIdAuditPrefix() + " delete " + name); // Execute the operation synchronously - wait for the operation to complete before continuing. - long procId = this.procedureExecutor.submitProcedure( - new DeleteNamespaceProcedure(procedureExecutor.getEnvironment(), name), - nonceGroup, - nonce); - ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId); - if (cpHost != null) { - cpHost.postDeleteNamespace(name); - } + ClusterSchema cs = getClusterSchema(); + cs.await(cs.deleteNamespace(name, nonceGroup, nonce)); + if (this.cpHost != null) this.cpHost.postDeleteNamespace(name); } /** - * Ensure that the specified namespace exists, otherwise throws a NamespaceNotFoundException - * - * @param name the namespace to check - * @throws IOException if the namespace manager is not ready yet. - * @throws NamespaceNotFoundException if the namespace does not exists + * Get a Namespace + * @param name Name of the Namespace + * @return Namespace descriptor for name + * @throws IOException */ - private void ensureNamespaceExists(final String name) - throws IOException, NamespaceNotFoundException { - checkNamespaceManagerReady(); - NamespaceDescriptor nsd = tableNamespaceManager.get(name); - if (nsd == null) { - throw new NamespaceNotFoundException(name); - } + NamespaceDescriptor getNamespace(String name) throws IOException { + checkInitialized(); + if (this.cpHost != null) this.cpHost.preGetNamespaceDescriptor(name); + NamespaceDescriptor nsd = this.clusterSchemaService.getNamespace(name); + if (this.cpHost != null) this.cpHost.postGetNamespaceDescriptor(nsd); + return nsd; } - @Override - public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException { - checkNamespaceManagerReady(); - + /** + * Get all Namespaces + * @return All Namespace descriptors + * @throws IOException + */ + List getNamespaces() throws IOException { + checkInitialized(); + final List nsds = new ArrayList(); + boolean bypass = false; if (cpHost != null) { - cpHost.preGetNamespaceDescriptor(name); - } - - NamespaceDescriptor nsd = tableNamespaceManager.get(name); - if (nsd == null) { - throw new NamespaceNotFoundException(name); + bypass = cpHost.preListNamespaceDescriptors(nsds); } - - if (cpHost != null) { - cpHost.postGetNamespaceDescriptor(nsd); + if (!bypass) { + nsds.addAll(this.clusterSchemaService.getNamespaces()); + if (this.cpHost != null) this.cpHost.postListNamespaceDescriptors(nsds); } - - return nsd; + return nsds; } @Override - public List listNamespaceDescriptors() throws IOException { - checkNamespaceManagerReady(); - - final List descriptors = new ArrayList(); - boolean bypass = false; - if (cpHost != null) { - bypass = cpHost.preListNamespaceDescriptors(descriptors); - } - - if (!bypass) { - descriptors.addAll(tableNamespaceManager.list()); + public List listTableNamesByNamespace(String name) throws IOException { + checkInitialized(); + return listTableNames(name, null, true); + } - if (cpHost != null) { - cpHost.postListNamespaceDescriptors(descriptors); - } - } - return descriptors; + @Override + public List listTableDescriptorsByNamespace(String name) throws IOException { + checkInitialized(); + return listTableDescriptors(name, null, null, true); } @Override @@ -2611,18 +2567,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return procInfoList; } - @Override - public List listTableDescriptorsByNamespace(String name) throws IOException { - ensureNamespaceExists(name); - return listTableDescriptors(name, null, null, true); - } - - @Override - public List listTableNamesByNamespace(String name) throws IOException { - ensureNamespaceExists(name); - return listTableNames(name, null, true); - } - /** * Returns the list of table descriptors that match the specified request * @@ -2645,13 +2589,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { if (!bypass) { if (tableNameList == null || tableNameList.size() == 0) { // request for all TableDescriptors - Collection htds; - if (namespace != null && namespace.length() > 0) { - htds = tableDescriptors.getByNamespace(namespace).values(); - } else { - htds = tableDescriptors.getAll().values(); - } - + Collection htds = getAllDescriptors(namespace); for (HTableDescriptor desc: htds) { if (tableStateManager.isTablePresent(desc.getTableName()) && (includeSysTables || !desc.getTableName().isSystemTable())) { @@ -2699,13 +2637,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { if (!bypass) { // get all descriptors - Collection htds; - if (namespace != null && namespace.length() > 0) { - htds = tableDescriptors.getByNamespace(namespace).values(); - } else { - htds = tableDescriptors.getAll().values(); - } - + Collection htds = getAllDescriptors(namespace); for (HTableDescriptor htd: htds) { if (includeSysTables || !htd.getTableName().isSystemTable()) { descriptors.add(htd); @@ -2729,6 +2661,18 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return result; } + private Collection getAllDescriptors(final String namespace) + throws IOException { + // get all descriptors + Collection htds; + if (namespace != null && namespace.length() > 0) { + if (getNamespace(namespace) == null) throw new NamespaceNotFoundException(namespace); + htds = this.tableDescriptors.getByNamespace(namespace).values(); + } else { + htds = this.tableDescriptors.getAll().values(); + } + return htds; + } /** * Removes the table descriptors that don't match the pattern. @@ -2865,4 +2809,4 @@ public class HMaster extends HRegionServer implements MasterServices, Server { public RegionNormalizerTracker getRegionNormalizerTracker() { return regionNormalizerTracker; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index b269c3d..801da82 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.NamespaceExistException; import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.ServerLoad; @@ -832,7 +833,7 @@ public class MasterRpcServices extends RSRpcServices try { return GetNamespaceDescriptorResponse.newBuilder() .setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor( - master.getNamespaceDescriptor(request.getNamespaceName()))) + master.getNamespace(request.getNamespaceName()))) .build(); } catch (IOException e) { throw new ServiceException(e); @@ -1120,7 +1121,7 @@ public class MasterRpcServices extends RSRpcServices try { ListNamespaceDescriptorsResponse.Builder response = ListNamespaceDescriptorsResponse.newBuilder(); - for(NamespaceDescriptor ns: master.listNamespaceDescriptors()) { + for(NamespaceDescriptor ns: master.getNamespaces()) { response.addNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(ns)); } return response.build(); @@ -1307,8 +1308,12 @@ public class MasterRpcServices extends RSRpcServices // ensure namespace exists TableName dstTable = TableName.valueOf(request.getSnapshot().getTable()); - master.getNamespaceDescriptor(dstTable.getNamespaceAsString()); - + NamespaceDescriptor nsd = NamespaceDescriptor.create(dstTable.getNamespaceAsString()).build(); + try { + master.createNamespace(nsd, HConstants.NO_NONCE, HConstants.NO_NONCE); + } catch (NamespaceExistException nee) { + if (LOG.isDebugEnabled()) LOG.debug("Namespace " + nsd.getName() + " exists"); + } SnapshotDescription reqSnapshot = request.getSnapshot(); master.snapshotManager.restoreSnapshot(reqSnapshot); return RestoreSnapshotResponse.newBuilder().build(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index 935644f..9f1f598 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -21,20 +21,19 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; import java.util.List; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; -import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.quotas.MasterQuotaManager; import com.google.protobuf.Service; @@ -45,6 +44,11 @@ import com.google.protobuf.Service; @InterfaceAudience.Private public interface MasterServices extends Server { /** + * @return Master's instance of {@link ClusterSchema} + */ + ClusterSchema getClusterSchema(); + + /** * @return Master's instance of the {@link AssignmentManager} */ AssignmentManager getAssignmentManager(); @@ -80,11 +84,6 @@ public interface MasterServices extends Server { MasterCoprocessorHost getMasterCoprocessorHost(); /** - * @return Master's instance of {@link TableNamespaceManager} - */ - TableNamespaceManager getTableNamespaceManager(); - - /** * @return Master's instance of {@link MasterQuotaManager} */ MasterQuotaManager getMasterQuotaManager(); @@ -274,54 +273,6 @@ public interface MasterServices extends Server { boolean isInitialized(); /** - * Create a new namespace - * @param descriptor descriptor which describes the new namespace - * @param nonceGroup - * @param nonce - * @throws IOException - */ - public void createNamespace( - final NamespaceDescriptor descriptor, - final long nonceGroup, - final long nonce) throws IOException; - - /** - * Create a new namespace synchronously. - * @param descriptor descriptor which describes the new namespace - * @param nonceGroup - * @param nonce - * @throws IOException - */ - public void createNamespaceSync( - final NamespaceDescriptor descriptor, - final long nonceGroup, - final long nonce) throws IOException; - - /** - * Modify an existing namespace - * @param descriptor descriptor which updates the existing namespace - * @param nonceGroup - * @param nonce - * @throws IOException - */ - public void modifyNamespace( - final NamespaceDescriptor descriptor, - final long nonceGroup, - final long nonce) throws IOException; - - /** - * Delete an existing namespace. Only empty namespaces (no tables) can be removed. - * @param name namespace name - * @param nonceGroup - * @param nonce - * @throws IOException - */ - public void deleteNamespace( - final String name, - final long nonceGroup, - final long nonce) throws IOException; - - /** * Abort a procedure. * @param procId ID of the procedure * @param mayInterruptIfRunning if the proc completed at least one step, should it be aborted? @@ -332,21 +283,6 @@ public interface MasterServices extends Server { throws IOException; /** - * Get a namespace descriptor by name - * @param name name of namespace descriptor - * @return A descriptor - * @throws IOException - */ - public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException; - - /** - * List available namespace descriptors - * @return A descriptor - * @throws IOException - */ - public List listNamespaceDescriptors() throws IOException; - - /** * List procedures * @return procedure list * @throws IOException diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java new file mode 100644 index 0000000..5bd8eee --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.hadoop.hbase.ProcedureInfo; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; + +/** + * Map Future Interface on to Procedure result processing. + */ +// Has no extra methods as of now beyond Future. Use #toString if you want to log +// procId of procedure. +// TODO: This should be in Procedure? Have it in master package for now. Lets out ProcedureInfo. +// Implementation informed by HBaseAdmin#ProcedureFuture. +@InterfaceAudience.Private +class ProcedureFuture implements Future { + // Save exception so we can rethrow if called again. Same for result. + private ExecutionException exception = null; + private ProcedureInfo result = null; + private boolean done = false; + private boolean cancelled = false; + private final Long procId; + private final ProcedureExecutor procedureExecutor; + + ProcedureFuture(final ProcedureExecutor procedureExecutor, + final long procId) { + this.procedureExecutor = procedureExecutor; + this.procId = procId; + } + + @Override + public String toString() { + return "procId=" + this.procId; + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + if (!this.cancelled) { + this.cancelled = this.procedureExecutor.abort(this.procId, mayInterruptIfRunning); + } + return this.cancelled; + } + + @Override + public boolean isCancelled() { + return this.cancelled; + } + + @Override + public boolean isDone() { + return done; + } + + @Override + public ProcedureInfo get() throws InterruptedException, ExecutionException { + // TODO: should we ever spin forever? + throw new UnsupportedOperationException(); + } + + @Override + public ProcedureInfo get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + if (!this.done) { + // TODO: add this sort of facility to EnvironmentEdgeManager + long deadlineTs = EnvironmentEdgeManager.currentTime() + unit.toMillis(timeout); + try { + this.result = waitProcedureResult(procId, deadlineTs); + } catch (IOException e) { + this.exception = new ExecutionException(e); + } + this.done = true; + } + if (exception != null) { + throw exception; + } + return result; + } + + /** + * @param procId + * @param deadlineTs + * @return A ProcedureInfo instance or null if procedure not found. + * @throws IOException + * @throws TimeoutException + * @throws InterruptedException + */ + private ProcedureInfo waitProcedureResult(long procId, long deadlineTs) + throws IOException, TimeoutException, InterruptedException { + while (EnvironmentEdgeManager.currentTime() < deadlineTs) { + Pair pair = this.procedureExecutor.getResultOrProcedure(procId); + if (pair.getFirst() != null) { + this.procedureExecutor.removeResult(procId); + return pair.getFirst(); + } else { + if (pair.getSecond() == null) return null; + } + // TODO: Add a wait. + } + throw new TimeoutException("The procedure " + procId + " is still running"); + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java index 4b36f59..c325221 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java @@ -27,17 +27,17 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZKNamespaceManager; -import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; @@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.constraint.ConstraintException; -import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure; import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; @@ -56,10 +55,11 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import com.google.common.collect.Sets; /** - * This is a helper class used to manage the namespace - * metadata that is stored in TableName.NAMESPACE_TABLE_NAME - * It also mirrors updates to the ZK store by forwarding updates to - * {@link org.apache.hadoop.hbase.ZKNamespaceManager} + * This is a helper class used internally to manage the namespace metadata that is stored in + * TableName.NAMESPACE_TABLE_NAME. It also mirrors updates to the ZK store by forwarding updates to + * {@link org.apache.hadoop.hbase.ZKNamespaceManager}. + * + * Do not use. Go via the higher-level {@link ClusterSchema} API instead. */ @InterfaceAudience.Private public class TableNamespaceManager { @@ -90,10 +90,9 @@ public class TableNamespaceManager { private long exclusiveLockTimeoutMs; private long sharedLockTimeoutMs; - public TableNamespaceManager(MasterServices masterServices) { + TableNamespaceManager(MasterServices masterServices) { this.masterServices = masterServices; this.conf = masterServices.getConfiguration(); - this.exclusiveLockTimeoutMs = conf.getLong( TABLE_WRITE_LOCK_TIMEOUT_MS, DEFAULT_TABLE_WRITE_LOCK_TIMEOUT_MS); @@ -113,7 +112,7 @@ public class TableNamespaceManager { // Wait for the namespace table to be initialized. long startTime = EnvironmentEdgeManager.currentTime(); int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT); - while (!isTableAvailableAndInitialized(false)) { + while (!isTableAvailableAndInitialized()) { if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) { // We can't do anything if ns is not online. throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " @@ -269,16 +268,28 @@ public class TableNamespaceManager { } /** + * Create Namespace in a blocking manner; don't return till success. + * Note, by-passes notifying coprocessors and name checks. Use for system namespaces only. + * @throws IOException + * @throws InterruptedException + */ + private void createSystemNamespace(final NamespaceDescriptor namespaceDescriptor) + throws IOException { + ClusterSchema cs = this.masterServices.getClusterSchema(); + cs.await(cs.createNamespace(namespaceDescriptor, HConstants.NO_NONCE, HConstants.NO_NONCE)); + } + + /** * This method checks if the namespace table is assigned and then - * tries to create its HTable. If it was already created before, it also makes + * tries to create its Table reference. If it was already created before, it also makes * sure that the connection isn't closed. * @return true if the namespace table manager is ready to serve, false * otherwise * @throws IOException */ @SuppressWarnings("deprecation") - public synchronized boolean isTableAvailableAndInitialized( - final boolean createNamespaceAync) throws IOException { + public synchronized boolean isTableAvailableAndInitialized() + throws IOException { // Did we already get a table? If so, still make sure it's available if (isTableNamespaceManagerInitialized()) { return true; @@ -293,34 +304,10 @@ public class TableNamespaceManager { zkNamespaceManager.start(); if (get(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE.getName()) == null) { - if (createNamespaceAync) { - masterServices.getMasterProcedureExecutor().submitProcedure( - new CreateNamespaceProcedure( - masterServices.getMasterProcedureExecutor().getEnvironment(), - NamespaceDescriptor.DEFAULT_NAMESPACE)); - initGoodSofar = false; - } - else { - masterServices.createNamespaceSync( - NamespaceDescriptor.DEFAULT_NAMESPACE, - HConstants.NO_NONCE, - HConstants.NO_NONCE); - } + createSystemNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE); } if (get(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE.getName()) == null) { - if (createNamespaceAync) { - masterServices.getMasterProcedureExecutor().submitProcedure( - new CreateNamespaceProcedure( - masterServices.getMasterProcedureExecutor().getEnvironment(), - NamespaceDescriptor.SYSTEM_NAMESPACE)); - initGoodSofar = false; - } - else { - masterServices.createNamespaceSync( - NamespaceDescriptor.SYSTEM_NAMESPACE, - HConstants.NO_NONCE, - HConstants.NO_NONCE); - } + createSystemNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE); } if (!initGoodSofar) { @@ -410,4 +397,4 @@ public class TableNamespaceManager { } return maxRegions; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java index 657bbfb..f934737 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java @@ -348,7 +348,7 @@ public class CreateNamespaceProcedure } private static TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) { - return env.getMasterServices().getTableNamespaceManager(); + return env.getMasterServices().getClusterSchema().getTableNamespaceManager(); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java index 5a42614..2f99167 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java @@ -383,7 +383,7 @@ public class DeleteNamespaceProcedure } private static TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) { - return env.getMasterServices().getTableNamespaceManager(); + return env.getMasterServices().getClusterSchema().getTableNamespaceManager(); } /** * The procedure could be restarted from a different machine. If the variable is null, we need to diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java index 30de252..0f8c172 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java @@ -266,8 +266,9 @@ public class ModifyNamespaceProcedure } private TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) { - return env.getMasterServices().getTableNamespaceManager(); + return env.getMasterServices().getClusterSchema().getTableNamespaceManager(); } + /** * The procedure could be restarted from a different machine. If the variable is null, we need to * retrieve it. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java index f24f8c0..8035d32 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java @@ -65,7 +65,7 @@ class NamespaceStateManager { /** * Gets an instance of NamespaceTableAndRegionInfo associated with namespace. - * @param The name of the namespace + * @param name The name of the namespace * @return An instance of NamespaceTableAndRegionInfo. */ public NamespaceTableAndRegionInfo getState(String name) { @@ -135,7 +135,7 @@ class NamespaceStateManager { private NamespaceDescriptor getNamespaceDescriptor(String namespaceAsString) { try { - return this.master.getNamespaceDescriptor(namespaceAsString); + return this.master.getClusterSchema().getNamespace(namespaceAsString); } catch (IOException e) { LOG.error("Error while fetching namespace descriptor for namespace : " + namespaceAsString); return null; @@ -212,7 +212,7 @@ class NamespaceStateManager { * Initialize namespace state cache by scanning meta table. */ private void initialize() throws IOException { - List namespaces = this.master.listNamespaceDescriptors(); + List namespaces = this.master.getClusterSchema().getNamespaces(); for (NamespaceDescriptor namespace : namespaces) { addNamespace(namespace.getName()); List tables = this.master.listTableNamesByNamespace(namespace.getName()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java index c24d8a3..d1ed966 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index b5f20f5..1b2d8a1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaMockingUtil; -import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.Server; @@ -142,11 +141,10 @@ public class TestCatalogJanitor { ServerName.valueOf("example.org,12345,6789"), HRegionInfo.FIRST_META_REGIONINFO); // Set hbase.rootdir into test dir. - FileSystem fs = FileSystem.get(this.c); + FileSystem.get(this.c); Path rootdir = FSUtils.getRootDir(this.c); FSUtils.setRootDir(this.c, rootdir); - AdminProtos.AdminService.BlockingInterface hri = - Mockito.mock(AdminProtos.AdminService.BlockingInterface.class); + Mockito.mock(AdminProtos.AdminService.BlockingInterface.class); } @Override @@ -396,48 +394,6 @@ public class TestCatalogJanitor { } @Override - public void createNamespace( - final NamespaceDescriptor descriptor, - final long nonceGroup, - final long nonce) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public void createNamespaceSync( - final NamespaceDescriptor descriptor, - final long nonceGroup, - final long nonce) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public void modifyNamespace( - final NamespaceDescriptor descriptor, - final long nonceGroup, - final long nonce) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public void deleteNamespace( - final String name, - final long nonceGroup, - final long nonce) throws IOException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException { - return null; //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - public List listNamespaceDescriptors() throws IOException { - return null; //To change body of implemented methods use File | Settings | File Templates. - } - - @Override public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning) throws IOException { return false; //To change body of implemented methods use File | Settings | File Templates. @@ -530,11 +486,6 @@ public class TestCatalogJanitor { } @Override - public TableNamespaceManager getTableNamespaceManager() { - return null; - } - - @Override public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b, boolean forcible) throws IOException { } @@ -556,6 +507,12 @@ public class TestCatalogJanitor { // Auto-generated method stub return 0; } + + @Override + public ClusterSchema getClusterSchema() { + // TODO Auto-generated method stub + return null; + } } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java index 972834a..20b492c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java @@ -218,10 +218,6 @@ public class TestMasterNoCluster { return null; } } - - @Override - void initNamespace() { - } }; master.start(); @@ -294,10 +290,6 @@ public class TestMasterNoCluster { return null; } } - - @Override - void initNamespace() { - } }; master.start(); -- 2.6.1