diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/BulkTest.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/BulkTest.java deleted file mode 100644 index e79b38a..0000000 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/BulkTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.jackrabbit.oak.segment.standby; - -import static junit.framework.Assert.assertNotNull; -import static junit.framework.Assert.assertTrue; -import static org.junit.Assert.assertEquals; - -import java.lang.management.ManagementFactory; -import java.util.Set; - -import javax.management.MBeanServer; -import javax.management.ObjectName; - -import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; -import org.apache.jackrabbit.oak.segment.standby.client.StandbyClient; -import org.apache.jackrabbit.oak.segment.standby.jmx.StandbyStatusMBean; -import org.apache.jackrabbit.oak.segment.standby.server.StandbyServer; -import org.apache.jackrabbit.oak.spi.commit.CommitInfo; -import org.apache.jackrabbit.oak.spi.commit.EmptyHook; -import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -public class BulkTest extends TestBase { - - @Before - public void setUp() throws Exception { - setUpServerAndClient(); - } - - @After - public void after() { - closeServerAndClient(); - } - - @Test - @Ignore("OAK-4707") - public void test100Nodes() throws Exception { - test(100, 1, 1, 3000, 3100); - } - - @Test - @Ignore("OAK-4707") - public void test1000Nodes() throws Exception { - test(1000, 1, 1, 53000, 55000); - } - - @Test - @Ignore("OAK-4707") - public void test10000Nodes() throws Exception { - test(10000, 1, 1, 245000, 246000); - } - - @Test - @Ignore("OAK-4707") - public void test100000Nodes() throws Exception { - test(100000, 9, 9, 2210000, 2220000); - } - - @Test - @Ignore("OAK-4707") - public void test1MillionNodes() throws Exception { - test(1000000, 87, 87, 22700000, 22800000); - } - - @Test - @Ignore("OAK-4707") - public void test1MillionNodesUsingSSL() throws Exception { - test(1000000, 87, 87, 22700000, 22800000, true); - } - -/* - @Test - public void test10MillionNodes() throws Exception { - test(10000000, 856, 856, 223000000, 224000000); - } -*/ - - // private helper - - private void test(int number, int minExpectedSegments, int maxExpectedSegments, long minExpectedBytes, long maxExpectedBytes) throws Exception { - test(number, minExpectedSegments, maxExpectedSegments, minExpectedBytes, maxExpectedBytes, false); - } - - private void test(int number, int minExpectedSegments, int maxExpectedSegments, long minExpectedBytes, long maxExpectedBytes, - boolean useSSL) throws Exception { - NodeStore store = SegmentNodeStoreBuilders.builder(storeS).build(); - NodeBuilder rootbuilder = store.getRoot().builder(); - NodeBuilder b = rootbuilder.child("store"); - for (int j=0; j<=number / 1000; j++) { - NodeBuilder builder = b.child("Folder#" + j); - for (int i = 0; i <(number < 1000 ? number : 1000); i++) { - builder.child("Test#" + i).setProperty("ts", System.currentTimeMillis()); - } - } - store.merge(rootbuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); - storeS.flush(); - - final StandbyServer server = new StandbyServer(port, storeS, useSSL); - server.start(); - - System.setProperty(StandbyClient.CLIENT_ID_PROPERTY_NAME, "Bar"); - StandbyClient cl = newStandbyClient(storeC, port, useSSL); - - final MBeanServer jmxServer = ManagementFactory.getPlatformMBeanServer(); - ObjectName status = new ObjectName(StandbyStatusMBean.JMX_NAME + ",id=*"); - ObjectName clientStatus = new ObjectName(cl.getMBeanName()); - ObjectName serverStatus = new ObjectName(server.getMBeanName()); - - long start = System.currentTimeMillis(); - cl.run(); - - try { - Set instances = jmxServer.queryNames(status, null); - assertEquals(3, instances.size()); - - ObjectName connectionStatus = null; - for (ObjectName s : instances) { - if (!s.equals(clientStatus) && !s.equals(serverStatus)) connectionStatus = s; - } - assertNotNull(connectionStatus); - - long segments = ((Long)jmxServer.getAttribute(connectionStatus, "TransferredSegments")).longValue(); - long bytes = ((Long)jmxServer.getAttribute(connectionStatus, "TransferredSegmentBytes")).longValue(); - - System.out.println("did transfer " + segments + " segments with " + bytes + " bytes in " + (System.currentTimeMillis() - start) / 1000 + " seconds."); - - assertEquals(storeS.getHead(), storeC.getHead()); - - //compare(segments, "segment", minExpectedSegments, maxExpectedSegments); - //compare(bytes, "byte", minExpectedBytes, maxExpectedBytes); - - } finally { - server.close(); - cl.close(); - } - } - - private void compare(long current, String unit, long expectedMin, long expectedMax) { - assertTrue("current number of " + unit + "s (" + current + ") is less than minimum expected: " + expectedMin, current >= expectedMin); - assertTrue("current number of " + unit + "s (" + current + ") is bigger than maximum expected: " + expectedMax, current <= expectedMax); - } -} diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTest.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTest.java index 3f65415..6bfdec0 100644 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTest.java +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTest.java @@ -28,7 +28,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.Random; -import com.google.common.io.ByteStreams; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; @@ -43,9 +42,10 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; +import com.google.common.io.ByteStreams; + public class StandbyTest extends TestBase { @Before diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTestIT.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTestIT.java index c34cbd0..6d9ad34 100644 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTestIT.java +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTestIT.java @@ -43,7 +43,6 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; public class StandbyTestIT extends TestBase { diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/TestBase.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/TestBase.java index 2c40490..21d19cb 100644 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/TestBase.java +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/TestBase.java @@ -37,7 +37,6 @@ import org.junit.Rule; import org.junit.rules.TemporaryFolder; public class TestBase { - static final int port = Integer.getInteger("standby.server.port", 52800); diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BenchmarkBase.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BenchmarkBase.java new file mode 100644 index 0000000..49e435a --- /dev/null +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BenchmarkBase.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment.standby.benchmark; + +import static java.io.File.createTempFile; +import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.standby.client.StandbyClient; +import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider; + +public class BenchmarkBase { + static final int port = Integer.getInteger("standby.server.port", 52800); + static final String LOCALHOST = "127.0.0.1"; + + static final int timeout = Integer.getInteger("standby.test.timeout", 500); + + File directoryS; + FileStore storeS; + ScheduledExecutorService executorS; + + File directoryC; + FileStore storeC; + ScheduledExecutorService executorC; + + public void setUpServerAndClient() throws Exception { + directoryS = createTmpTargetDir(getClass().getSimpleName() + "-Server"); + executorS = Executors.newSingleThreadScheduledExecutor(); + storeS = setupPrimary(directoryS, executorS); + + // client + directoryC = createTmpTargetDir(getClass().getSimpleName() + "-Client"); + executorC = Executors.newSingleThreadScheduledExecutor(); + storeC = setupSecondary(directoryC, executorC); + } + + public void closeServerAndClient() { + storeS.close(); + storeC.close(); + + try { + FileUtils.deleteDirectory(directoryS); + FileUtils.deleteDirectory(directoryC); + } catch (IOException e) { + // ignore + } finally { + if (executorS != null) { + new ExecutorCloser(executorS).close(); + } + + if (executorC != null) { + new ExecutorCloser(executorC).close(); + } + } + } + + private static FileStore newFileStore(File directory, ScheduledExecutorService executor) throws Exception { + return fileStoreBuilder(directory).withMaxFileSize(1).withMemoryMapping(false).withNodeDeduplicationCacheSize(0) + .withSegmentCacheSize(0).withStringCacheSize(0).withTemplateCacheSize(0) + .withStatisticsProvider(new DefaultStatisticsProvider(executor)).build(); + } + + protected FileStore setupPrimary(File directory, ScheduledExecutorService executor) throws Exception { + return newFileStore(directory, executor); + } + + protected FileStore setupSecondary(File directory, ScheduledExecutorService executor) throws Exception { + return newFileStore(directory, executor); + } + + public StandbyClient newStandbyClient(FileStore store) throws Exception { + return newStandbyClient(store, port, false); + } + + public StandbyClient newStandbyClient(FileStore store, int port) throws Exception { + return newStandbyClient(store, port, false); + } + + public StandbyClient newStandbyClient(FileStore store, int port, boolean secure) throws Exception { + return new StandbyClient(LOCALHOST, port, store, secure, timeout, false); + } + + private static File createTmpTargetDir(String name) throws IOException { + File f = createTempFile(name, "dir", new File("target")); + f.delete(); + f.mkdir(); + return f; + } +} diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BulkTransferBenchmark.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BulkTransferBenchmark.java new file mode 100644 index 0000000..0f2dfc6 --- /dev/null +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BulkTransferBenchmark.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.standby.benchmark; + +import java.lang.management.ManagementFactory; +import java.lang.reflect.Method; +import java.util.Set; + +import javax.management.MBeanServer; +import javax.management.ObjectName; + +import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; +import org.apache.jackrabbit.oak.segment.standby.client.StandbyClient; +import org.apache.jackrabbit.oak.segment.standby.jmx.StandbyStatusMBean; +import org.apache.jackrabbit.oak.segment.standby.server.StandbyServer; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; + +public class BulkTransferBenchmark extends BenchmarkBase { + + public void setUp() throws Exception { + setUpServerAndClient(); + } + + public void after() { + closeServerAndClient(); + } + + public void test100Nodes() throws Exception { + test(100, 1, 1, 3000, 3100); + } + + public void test1000Nodes() throws Exception { + test(1000, 1, 1, 53000, 55000); + } + + public void test10000Nodes() throws Exception { + test(10000, 1, 1, 245000, 246000); + } + + public void test100000Nodes() throws Exception { + test(100000, 9, 9, 2210000, 2220000); + } + + public void test1MillionNodes() throws Exception { + test(1000000, 87, 87, 22700000, 22800000); + } + + public void test1MillionNodesUsingSSL() throws Exception { + test(1000000, 87, 87, 22700000, 22800000, true); + } + + public void test10MillionNodes() throws Exception { + test(10000000, 856, 856, 223000000, 224000000); + } + + private void test(int number, int minExpectedSegments, int maxExpectedSegments, long minExpectedBytes, long maxExpectedBytes) throws Exception { + test(number, minExpectedSegments, maxExpectedSegments, minExpectedBytes, maxExpectedBytes, false); + } + + private void test(int number, int minExpectedSegments, int maxExpectedSegments, long minExpectedBytes, long maxExpectedBytes, + boolean useSSL) throws Exception { + NodeStore store = SegmentNodeStoreBuilders.builder(storeS).build(); + NodeBuilder rootbuilder = store.getRoot().builder(); + NodeBuilder b = rootbuilder.child("store"); + for (int j=0; j<=number / 1000; j++) { + NodeBuilder builder = b.child("Folder#" + j); + for (int i = 0; i <(number < 1000 ? number : 1000); i++) { + builder.child("Test#" + i).setProperty("ts", System.currentTimeMillis()); + } + } + store.merge(rootbuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + storeS.flush(); + + final StandbyServer server = new StandbyServer(port, storeS, useSSL); + server.start(); + + System.setProperty(StandbyClient.CLIENT_ID_PROPERTY_NAME, "Bar"); + StandbyClient cl = newStandbyClient(storeC, port, useSSL); + + final MBeanServer jmxServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName status = new ObjectName(StandbyStatusMBean.JMX_NAME + ",id=*"); + ObjectName clientStatus = new ObjectName(cl.getMBeanName()); + ObjectName serverStatus = new ObjectName(server.getMBeanName()); + + long start = System.currentTimeMillis(); + cl.run(); + + try { + Set instances = jmxServer.queryNames(status, null); + ObjectName connectionStatus = null; + for (ObjectName s : instances) { + if (!s.equals(clientStatus) && !s.equals(serverStatus)) connectionStatus = s; + } + assert(connectionStatus != null); + + long segments = ((Long)jmxServer.getAttribute(connectionStatus, "TransferredSegments")).longValue(); + long bytes = ((Long)jmxServer.getAttribute(connectionStatus, "TransferredSegmentBytes")).longValue(); + + System.out.println("did transfer " + segments + " segments with " + bytes + " bytes in " + (System.currentTimeMillis() - start) / 1000 + " seconds."); + } finally { + server.close(); + cl.close(); + } + } + + public static void main(String[] args) { + BulkTransferBenchmark benchmark = new BulkTransferBenchmark(); + + String[] methodNames = new String[] { + "test100Nodes", + "test1000Nodes", + "test10000Nodes", + "test100000Nodes", + "test1MillionNodes", + "test1MillionNodesUsingSSL", + "test10MillionNodes" + }; + + for (String methodName : methodNames) { + try { + Method method = benchmark.getClass().getMethod(methodName); + + benchmark.setUp(); + method.invoke(benchmark); + benchmark.after(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } +}