From 148f4c315254abfc60daf095543fe0ecc8ce44ba Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Mon, 6 Apr 2015 10:22:23 +0700 Subject: [PATCH] # ignite-564 --- .../cache/store/jdbc/CacheAbstractJdbcStore.java | 98 +++++++++++++++++----- .../yardstick/config/benchmark-store.properties | 5 +- 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java index 75e2080..df5345b 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheAbstractJdbcStore.java @@ -233,8 +233,12 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, if (dataSrc == null) throw new IgniteException("Failed to initialize cache store (data source is not provided)."); - if (dialect == null) + if (dialect == null) { dialect = resolveDialect(); + + if (log.isDebugEnabled() && dialect.getClass() != BasicJdbcDialect.class) + log.debug("Resolved database dialect: " + U.getSimpleName(dialect.getClass())); + } } /** {@inheritDoc} */ @@ -332,10 +336,10 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, finally { U.closeQuiet(conn); } - } - if (tx != null && log.isDebugEnabled()) - log.debug("Transaction ended [xid=" + tx.xid() + ", commit=" + commit + ']'); + if (log.isDebugEnabled()) + log.debug("Transaction ended [xid=" + tx.xid() + ", commit=" + commit + ']'); + } } /** @@ -616,6 +620,8 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, ExecutorService pool = null; try { + String cacheName = session().cacheName(); + pool = Executors.newFixedThreadPool(maxPoolSz); Collection> futs = new ArrayList<>(); @@ -627,8 +633,6 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, if (log.isDebugEnabled()) log.debug("Start loading entries from db using user queries from arguments"); - String cacheName = session().cacheName(); - for (int i = 0; i < args.length; i += 2) { String keyType = args[i].toString(); @@ -642,11 +646,11 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, else { Collection entryMappings = cacheMappings(session().cacheName()).values(); - if (log.isDebugEnabled()) - log.debug("Start loading all cache types entries from db"); - for (EntryMapping em : entryMappings) { if (parallelLoadCacheMinThreshold > 0) { + log.debug("Start parallel loading entries of one type from db [cache name=" + cacheName + + ", key type=" + em.keyType() + " ]"); + Connection conn = null; try { @@ -691,13 +695,21 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, U.closeQuiet(conn); } } - else + else { + if (log.isDebugEnabled()) + log.debug("Start loading entries of one type from db [cache name=" + cacheName + + ", key type=" + em.keyType() + " ]"); + futs.add(pool.submit(loadCacheFull(em, clo))); + } } } for (Future fut : futs) U.get(fut); + + if (log.isDebugEnabled()) + log.debug("Finished load cache: " + cacheName + " from db"); } catch (IgniteCheckedException e) { throw new CacheLoaderException("Failed to load cache", e.getCause()); @@ -936,10 +948,10 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, try { conn = connection(); - Object currKeyTypeId = null; - String cacheName = session().cacheName(); + Object currKeyTypeId = null; + if (dialect.hasMerge()) { PreparedStatement mergeStmt = null; @@ -963,6 +975,10 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, if (currKeyTypeId == null || !currKeyTypeId.equals(keyTypeId)) { if (mergeStmt != null) { + if (log.isDebugEnabled()) + log.debug("Start write entries to database [cache name=" + cacheName + + ", key type=" + em.keyType() + ", count=" + prepared + "]"); + executeBatch(em, mergeStmt, "writeAll", fromIdx, prepared, lazyEntries); U.closeQuiet(mergeStmt); @@ -972,6 +988,8 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, currKeyTypeId = keyTypeId; + fromIdx += prepared; + prepared = 0; } @@ -982,20 +1000,35 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, mergeStmt.addBatch(); if (++prepared % batchSz == 0) { + if (log.isDebugEnabled()) + log.debug("Start write entries to database [cache name=" + cacheName + + ", key type=" + em.keyType() + ", count=" + prepared + "]"); + executeBatch(em, mergeStmt, "writeAll", fromIdx, prepared, lazyEntries); + fromIdx += prepared; + prepared = 0; } } - if (mergeStmt != null && prepared % batchSz != 0) + if (mergeStmt != null && prepared % batchSz != 0) { + if (log.isDebugEnabled()) + log.debug("Start write entries to database [cache name=" + cacheName + + ", key type=" + em.keyType() + ", count=" + prepared + "]"); + executeBatch(em, mergeStmt, "writeAll", fromIdx, prepared, lazyEntries); + + } } finally { U.closeQuiet(mergeStmt); } } else { + log.debug("Start write entries to database by one using update and insert statement [cache name=" + + cacheName + ", count=" + entries.size() + "]"); + PreparedStatement insStmt = null; PreparedStatement updStmt = null; @@ -1129,22 +1162,22 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, try { conn = connection(); - Object currKeyTypeId = null; - - EntryMapping em = null; - - PreparedStatement delStmt = null; - LazyValue lazyKeys = new LazyValue() { @Override public Object[] create() { return keys.toArray(); } }; - int fromIdx = 0, prepared = 0; - String cacheName = session().cacheName(); + Object currKeyTypeId = null; + + EntryMapping em = null; + + PreparedStatement delStmt = null; + + int fromIdx = 0, prepared = 0; + for (Object key : keys) { Object keyTypeId = keyTypeId(key); @@ -1157,6 +1190,10 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, } if (!currKeyTypeId.equals(keyTypeId)) { + if (log.isDebugEnabled()) + log.debug("Start delete entries to database [cache name=" + cacheName + + ", key type=" + em.keyType() + ", count=" + prepared + "]"); + executeBatch(em, delStmt, "deleteAll", fromIdx, prepared, lazyKeys); fromIdx += prepared; @@ -1171,6 +1208,10 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, delStmt.addBatch(); if (++prepared % batchSz == 0) { + if (log.isDebugEnabled()) + log.debug("Start delete entries to database [cache name=" + cacheName + + ", key type=" + em.keyType() + ", count=" + prepared + "]"); + executeBatch(em, delStmt, "deleteAll", fromIdx, prepared, lazyKeys); fromIdx += prepared; @@ -1179,8 +1220,13 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, } } - if (delStmt != null && prepared % batchSz != 0) + if (delStmt != null && prepared % batchSz != 0) { + if (log.isDebugEnabled()) + log.debug("Start delete entries to database [cache name=" + cacheName + + ", key type=" + em.keyType() + ", count=" + prepared + "]"); + executeBatch(em, delStmt, "deleteAll", fromIdx, prepared, lazyKeys); + } } catch (SQLException e) { throw new CacheWriterException("Failed to remove values from database", e); @@ -1600,6 +1646,10 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, /** {@inheritDoc} */ @Override public Void call() throws Exception { + if (log.isDebugEnabled()) + log.debug("Start load cache using custom query [cache name= " + em.cacheName + + ", key type=" + em.keyType() + ", query=" + qry + "]"); + Connection conn = null; PreparedStatement stmt = null; @@ -1692,6 +1742,10 @@ public abstract class CacheAbstractJdbcStore implements CacheStore, /** {@inheritDoc} */ @Override public Map call() throws Exception { + if (log.isDebugEnabled()) + log.debug("Start load values from database [table= " + em.fullTableName() + + ", key count=" + keys.size() + "]"); + PreparedStatement stmt = null; try { diff --git a/modules/yardstick/config/benchmark-store.properties b/modules/yardstick/config/benchmark-store.properties index 8a7c265..cb9e507 100644 --- a/modules/yardstick/config/benchmark-store.properties +++ b/modules/yardstick/config/benchmark-store.properties @@ -77,17 +77,14 @@ CONFIGS="\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStoreGetBenchmark -sn IgniteNode -cs -wb -ds atomic-writeBehind-get,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStoreGetTxBenchmark -sn IgniteNode -ds tx-get,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStoreGetTxBenchmark -sn IgniteNode -cs -ds tx-store-get,\ --cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStoreGetTxBenchmark -sn IgniteNode -cs -wb -ds tx-writeBehind-get,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutBenchmark -sn IgniteNode -ds atomic-put,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutBenchmark -sn IgniteNode -cs -ds atomic-store-put,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutBenchmark -sn IgniteNode -cs -wb -ds atomic-writeBehind-put,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutTxBenchmark -sn IgniteNode -ds tx-put,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutTxBenchmark -sn IgniteNode -cs -ds tx-store-put,\ --cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutTxBenchmark -sn IgniteNode -cs -wb -ds tx-writeBehind-put,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutGetBenchmark -sn IgniteNode -ds atomic-put-get,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutGetBenchmark -sn IgniteNode -cs -ds atomic-store-put-get,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutGetBenchmark -sn IgniteNode -cs -wb -ds atomic-writeBehind-put-get,\ -cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutGetTxBenchmark -sn IgniteNode -ds tx-put-get,\ --cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutGetTxBenchmark -sn IgniteNode -cs -ds tx-store-put-get,\ --cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutGetTxBenchmark -sn IgniteNode -cs -wb -ds tx-writeBehind-put-get\ +-cfg ${SCRIPT_DIR}/../config/ignite-store-config.xml -nn ${nodesNum} -b 1 -w ${warm_up} -d ${run} -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcStorePutGetTxBenchmark -sn IgniteNode -cs -ds tx-store-put-get\ " -- 2.3.5