From 0cbaf6c51d6ba23e477fcc6aa0aed25c249ec486 Mon Sep 17 00:00:00 2001 From: Xiaobing Zhou Date: Thu, 20 Jul 2017 16:52:01 -0700 Subject: [PATCH] HBASE-18078. [C++] Harden RPC by handling various communication abnormalities --- .../connection/connection-factory.cc | 38 ++++++++--- .../connection/connection-factory.h | 13 +++- hbase-native-client/connection/connection-pool.cc | 75 ++++++++++++++++++---- hbase-native-client/connection/connection-pool.h | 7 ++ hbase-native-client/connection/rpc-client.cc | 17 +++-- hbase-native-client/connection/rpc-client.h | 5 +- hbase-native-client/exceptions/exception.h | 21 +++++- 7 files changed, 146 insertions(+), 30 deletions(-) diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc index a0c7f96118..8ddada93e7 100644 --- a/hbase-native-client/connection/connection-factory.cc +++ b/hbase-native-client/connection/connection-factory.cc @@ -27,6 +27,9 @@ #include "connection/pipeline.h" #include "connection/sasl-handler.h" #include "connection/service.h" +#include +#include +#include "exceptions/exception.h" using std::chrono::milliseconds; using std::chrono::nanoseconds; @@ -56,15 +59,30 @@ std::shared_ptr> ConnectionFactory::M std::shared_ptr ConnectionFactory::Connect( std::shared_ptr> client, const std::string &hostname, uint16_t port) { - // Yes this will block however it makes dealing with connection pool soooooo - // much nicer. - // TODO see about using shared promise for this. - auto pipeline = client - ->connect(folly::SocketAddress(hostname, port, true), - std::chrono::duration_cast(connect_timeout_)) - .get(); - auto dispatcher = std::make_shared(); - dispatcher->setPipeline(pipeline); - return dispatcher; + return AsyncConnect(client, hostname, port).get(); +} + +folly::Future> ConnectionFactory::AsyncConnect( + std::shared_ptr> client, const std::string &hostname, + uint16_t port) { + + folly::Promise> promise; + auto future = promise.getFuture(); + + try { + /* any connection error (e.g. timeout) will be folly::AsyncSocketException */ + auto pipeline = client->connect( + SocketAddress(hostname, port, true), + std::chrono::duration_cast(connect_timeout_)).get(); + auto dispatcher = std::make_shared(); + dispatcher->setPipeline(pipeline); + promise.setValue(dispatcher); + } catch(const folly::AsyncSocketException &e) { + promise.setException( + folly::make_exception_wrapper( + folly::make_exception_wrapper(e))); + } + + return future; } } // namespace hbase diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h index c96087d1dc..65e9327763 100644 --- a/hbase-native-client/connection/connection-factory.h +++ b/hbase-native-client/connection/connection-factory.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include @@ -55,7 +56,7 @@ class ConnectionFactory { virtual std::shared_ptr> MakeBootstrap(); /** - * Connect a ClientBootstrap to a server and return the pipeline. + * Connect a ClientBootstrap to a server and return the wangle::Service. * * This is mostly visible so that mocks can override socket connections. */ @@ -63,6 +64,16 @@ class ConnectionFactory { std::shared_ptr> client, const std::string &hostname, uint16_t port); + /** + * Asynchronously Connect a ClientBootstrap to a server and return the wangle::Service. + * + * This async function makes it easy to propagate exceptions in a controlled way with + * help of folly::Future/Promise. + */ + virtual folly::Future> AsyncConnect( + std::shared_ptr> client, + const std::string &hostname, uint16_t port); + private: std::chrono::nanoseconds connect_timeout_; std::shared_ptr conf_; diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc index e98759d2fc..3663ec59f2 100644 --- a/hbase-native-client/connection/connection-pool.cc +++ b/hbase-native-client/connection/connection-pool.cc @@ -22,11 +22,15 @@ #include #include #include +#include #include #include +#include "exceptions/exception.h" using std::chrono::nanoseconds; +using namespace folly; +using namespace hbase; namespace hbase { @@ -45,24 +49,31 @@ ConnectionPool::~ConnectionPool() { Close(); } std::shared_ptr ConnectionPool::GetConnection( std::shared_ptr remote_id) { - // Try and get th cached connection. - auto found_ptr = GetCachedConnection(remote_id); + /** + * Try and get the cached connection, if there's no connection then create it. + */ + auto found_ptr = GetCachedConnection(remote_id); + return found_ptr == nullptr ? GetNewConnection(remote_id) : found_ptr; +} - // If there's no connection then create it. - if (found_ptr == nullptr) { - found_ptr = GetNewConnection(remote_id); - } - return found_ptr; + +folly::Future> ConnectionPool::AsyncGetConnection( + std::shared_ptr remote_id) { + /** + * Try and get the cached connection, if there's no connection then create it. + */ + auto found_ptr = GetCachedConnection(remote_id); + return + found_ptr == nullptr ? + AsyncGetNewConnection(remote_id) + : folly::makeFuture>(std::move(found_ptr)); } std::shared_ptr ConnectionPool::GetCachedConnection( std::shared_ptr remote_id) { folly::SharedMutexWritePriority::ReadHolder holder(map_mutex_); auto found = connections_.find(remote_id); - if (found == connections_.end()) { - return nullptr; - } - return found->second; + return found == connections_.end() ? nullptr : found->second; } std::shared_ptr ConnectionPool::GetNewConnection( @@ -91,11 +102,51 @@ std::shared_ptr ConnectionPool::GetNewConnection( connections_.insert(std::make_pair(remote_id, connection)); clients_.insert(std::make_pair(remote_id, clientBootstrap)); - return connection; } } +folly::Future> ConnectionPool::AsyncGetNewConnection( + std::shared_ptr remote_id) { + // Grab the upgrade lock. While we are double checking other readers can + // continue on + SharedMutexWritePriority::UpgradeHolder u_holder{map_mutex_}; + + folly::Promise> promise; + auto future = promise.getFuture(); + + // Now check if someone else created the connection before we got the lock + // This is safe since we hold the upgrade lock. + // upgrade lock is more power than the reader lock. + auto found = connections_.find(remote_id); + if (found != connections_.end() && found->second != nullptr) { + promise.setValue(found->second); + } else { + // Yeah it looks a lot like there's no connection + SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)}; + + // Make double sure there are not stale connections hanging around. + connections_.erase(remote_id); + + /* create new connection */ + auto clientBootstrap = cf_->MakeBootstrap(); + try { + auto dispatcher = cf_->Connect(clientBootstrap, remote_id->host(), remote_id->port()); + auto connection = std::make_shared(remote_id, dispatcher); + promise.setValue(connection); + + connections_.insert(std::make_pair(remote_id, connection)); + clients_.insert(std::make_pair(remote_id, clientBootstrap)); + } catch(const hbase::ConnectionException &e) { + /* propagating ConnectionException up */ + promise.setException( + folly::make_exception_wrapper(e)); + } + } + + return future; +} + void ConnectionPool::Close(std::shared_ptr remote_id) { folly::SharedMutexWritePriority::WriteHolder holder{map_mutex_}; DLOG(INFO) << "Closing RPC Connection to host:" << remote_id->host() diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h index c7c4246e2a..e6f5c22539 100644 --- a/hbase-native-client/connection/connection-pool.h +++ b/hbase-native-client/connection/connection-pool.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include #include @@ -66,6 +67,11 @@ class ConnectionPool { std::shared_ptr GetConnection(std::shared_ptr remote_id); /** + * Asynchronously get connection by ConnectionId. + */ + folly::Future> AsyncGetConnection(std::shared_ptr remote_id); + + /** * Close/remove a connection. */ void Close(std::shared_ptr remote_id); @@ -78,6 +84,7 @@ class ConnectionPool { private: std::shared_ptr GetCachedConnection(std::shared_ptr remote_id); std::shared_ptr GetNewConnection(std::shared_ptr remote_id); + folly::Future> AsyncGetNewConnection(std::shared_ptr remote_id); std::unordered_map, std::shared_ptr, ConnectionIdHash, ConnectionIdEquals> connections_; diff --git a/hbase-native-client/connection/rpc-client.cc b/hbase-native-client/connection/rpc-client.cc index 10faa7a84e..43c363e5df 100644 --- a/hbase-native-client/connection/rpc-client.cc +++ b/hbase-native-client/connection/rpc-client.cc @@ -22,6 +22,7 @@ #include #include #include +#include "exceptions/exception.h" using hbase::security::User; using std::chrono::nanoseconds; @@ -55,7 +56,7 @@ folly::Future> RpcClient::AsyncCall(const std::string& std::unique_ptr req, std::shared_ptr ticket) { auto remote_id = std::make_shared(host, port, ticket); - return GetConnection(remote_id)->SendRequest(std::move(req)); + return CallForResult(remote_id, std::move(req)); } folly::Future> RpcClient::AsyncCall(const std::string& host, @@ -64,10 +65,18 @@ folly::Future> RpcClient::AsyncCall(const std::string& std::shared_ptr ticket, const std::string& service_name) { auto remote_id = std::make_shared(host, port, ticket, service_name); - return GetConnection(remote_id)->SendRequest(std::move(req)); + return CallForResult(remote_id, std::move(req)); } -std::shared_ptr RpcClient::GetConnection(std::shared_ptr remote_id) { - return cp_->GetConnection(remote_id); +folly::Future> RpcClient::CallForResult( + std::shared_ptr remote_id, + std::unique_ptr req) { + try { + auto connection = cp_->AsyncGetConnection(remote_id).get(); + return connection->SendRequest(std::move(req)); + } catch (const hbase::ConnectionException &e) { + return folly::makeFuture>( + folly::make_exception_wrapper(e)); + } } } // namespace hbase diff --git a/hbase-native-client/connection/rpc-client.h b/hbase-native-client/connection/rpc-client.h index 0ecde5b775..a615f523d2 100644 --- a/hbase-native-client/connection/rpc-client.h +++ b/hbase-native-client/connection/rpc-client.h @@ -63,8 +63,9 @@ class RpcClient { std::shared_ptr connection_pool() const { return cp_; } - private: - std::shared_ptr GetConnection(std::shared_ptr remote_id); +private: + folly::Future> CallForResult(std::shared_ptr remote_id, + std::unique_ptr req); private: std::shared_ptr cp_; diff --git a/hbase-native-client/exceptions/exception.h b/hbase-native-client/exceptions/exception.h index bdedff4068..d62807839b 100644 --- a/hbase-native-client/exceptions/exception.h +++ b/hbase-native-client/exceptions/exception.h @@ -59,7 +59,7 @@ class IOException : public std::logic_error { IOException(const std::string& what, bool do_not_retry) : logic_error(what), do_not_retry_(do_not_retry) {} - IOException(const std::string& what, folly::exception_wrapper cause) + IOException(const std::string& what, const folly::exception_wrapper &cause) : logic_error(what), cause_(cause), do_not_retry_(false) {} IOException(const std::string& what, folly::exception_wrapper cause, bool do_not_retry) @@ -115,6 +115,25 @@ class RetriesExhaustedException : public IOException { int32_t num_retries_; }; +class ConnectionException : public IOException { +public: + ConnectionException() { + } + + ConnectionException(const std::string& what) : + IOException(what) { + } + + ConnectionException(const folly::exception_wrapper &cause) : + IOException("", cause) { + } + + ConnectionException( + const std::string& what, + const folly::exception_wrapper &cause) : IOException(what, cause) { + } +}; + class RemoteException : public IOException { public: RemoteException() : IOException(), port_(0) {} -- 2.11.0 (Apple Git-81)