Index: oak-run/README.md IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-run/README.md (revision 145f23b3a42a746aca0d81003fdaa4efd90c748d) +++ oak-run/README.md (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) @@ -517,6 +517,7 @@ [--max-age ] \ [--verbose] \ [|] + [--metrics] [--export-metrics] The following operations are available: @@ -539,9 +540,32 @@ - Path to the tar segment store or the segment azure uri as specified in http://jackrabbit.apache.org/oak/docs/nodestore/segment/overview.html#remote-segment-stores or if Mongo NodeStore then the mongo uri. + --metrics - If metrics are to be captured. + --export-metrics - Option to export the captured metrics. The format of the command is type;URL;key1=value1,key2=value2 + Currently only [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) is supported + e.g. --export-metrics "pushgateway;localhost:9091;key1=value1,key2=value2" Note: +Note: When using --export-metrics the following additional jars have to be downloaded to support Prometheus Pushgatway +* [simpleclient_common-0.6.0.jar](http://central.maven.org/maven2/io/prometheus/simpleclient_common/0.6.0/simpleclient_common-0.6.0.jar) +* [simpleclient-0.6.0.jar](http://central.maven.org/maven2/io/prometheus/simpleclient/0.6.0/simpleclient-0.6.0.jar) +* [simpleclient_pushgateway-0.6.0.jar](http://central.maven.org/maven2/io/prometheus/simpleclient_pushgateway/0.6.0/simpleclient_pushgateway-0.6.0.jar) +* [simpleclient_dropwizard-0.6.0.jar](http://central.maven.org/maven2/io/prometheus/simpleclient_dropwizard/0.6.0/simpleclient_dropwizard-0.6.0.jar) + +The command to be executed when using this option is: + + $ java -classpath oak-run-*.jar:simpleclient_common-0.6.0.jar:simpleclient-0.6.0.jar:simpleclient_dropwizard-0.6.0.jar:simpleclient_pushgateway-0.6.0.jar \ + org.apache.jackrabbit.oak.run.Main \ + datastore [--check-consistency|--collect-garbage [true]] \ + [--s3ds |--fds |--azureds |fake-ds-path ] \ + [--out-dir ] \ + [--work-dir ] \ + [--max-age ] \ + [--verbose] \ + [|] + [--metrics] [--export-metrics] + Data Store and node store configuration is mandatory. The config files should be formatted according to the OSGi configuration admin specification Index: oak-run/pom.xml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-run/pom.xml (revision 145f23b3a42a746aca0d81003fdaa4efd90c748d) +++ oak-run/pom.xml (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) @@ -372,7 +372,28 @@ org.jetbrains annotations - + + + + io.prometheus + simpleclient + 0.6.0 + true + + + io.prometheus + simpleclient_dropwizard + 0.6.0 + true + + + + io.prometheus + simpleclient_pushgateway + 0.6.0 + true + + junit Index: oak-run/src/main/assembly/oak-run.xml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-run/src/main/assembly/oak-run.xml (revision 145f23b3a42a746aca0d81003fdaa4efd90c748d) +++ oak-run/src/main/assembly/oak-run.xml (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) @@ -34,6 +34,7 @@ org.apache.tika:tika-core:* org.apache.tika:tika-parsers:* org.apache.jackrabbit:jackrabbit-aws-ext:* + io.prometheus:simpleclient*:* true true Index: oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java (revision 145f23b3a42a746aca0d81003fdaa4efd90c748d) +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) @@ -151,16 +151,23 @@ private void execute(NodeStoreFixture fixture, DataStoreOptions dataStoreOpts, Options opts, Closer closer) throws Exception { - MarkSweepGarbageCollector collector = getCollector(fixture, dataStoreOpts, opts, closer); - if (dataStoreOpts.checkConsistency()) { - long missing = collector.checkConsistency(); - log.warn("Found {} missing blobs", missing); + + try (Closer metricsCloser = Closer.create()) { + MetricsExporterFixture metricsExporterFixture = + MetricsExporterFixtureProvider.create(dataStoreOpts, fixture.getWhiteboard()); + metricsCloser.register(metricsExporterFixture); + + MarkSweepGarbageCollector collector = getCollector(fixture, dataStoreOpts, opts, closer); + if (dataStoreOpts.checkConsistency()) { + long missing = collector.checkConsistency(); + log.warn("Found {} missing blobs", missing); - if (dataStoreOpts.isVerbose()) { - new VerboseIdLogger(opts).log(); - } - } else if (dataStoreOpts.collectGarbage()) { - collector.collectGarbage(dataStoreOpts.markOnly()); + if (dataStoreOpts.isVerbose()) { + new VerboseIdLogger(opts).log(); + } + } else if (dataStoreOpts.collectGarbage()) { + collector.collectGarbage(dataStoreOpts.markOnly()); + } } } Index: oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java (revision 145f23b3a42a746aca0d81003fdaa4efd90c748d) +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) @@ -47,6 +47,7 @@ private final OptionSpec blobGcMaxAgeInSecs; private final OptionSpec verbose; private final OptionSpec resetLoggingConfig; + private OptionSpec exportMetrics; public DataStoreOptions(OptionParser parser) { collectGarbage = parser.accepts("collect-garbage", @@ -73,6 +74,8 @@ resetLoggingConfig = parser.accepts("reset-log-config", "Reset logging config for testing purposes only").withOptionalArg() .ofType(Boolean.class).defaultsTo(Boolean.TRUE); + exportMetrics = parser.accepts("export-metrics", + "type, URI to export the metrics and optional metadata all delimeted by semi-colon(;)").withRequiredArg(); //Set of options which define action actionOpts = ImmutableSet.of(collectGarbage, consistencyCheck); @@ -158,4 +161,12 @@ } return result; } + + public boolean exportMetrics() { + return options.has(exportMetrics); + } + + public String exportMetricsArgs() { + return exportMetrics.value(options); + } } Index: oak-run/src/main/java/org/apache/jackrabbit/oak/run/MetricsExporterFixture.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/MetricsExporterFixture.java (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/MetricsExporterFixture.java (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.run; + +import java.io.Closeable; + +/** + * Fixture encapsulating metrics exporter instance of T + * @param + */ +public interface MetricsExporterFixture extends Closeable { + + ExporterType getExporterType(); + + T getMetricsExporter(); + + enum ExporterType { + pushgateway("Prometheus Push Gateway"); + + private String type; + + ExporterType(String type) { + this.type = type; + } + } +} Index: oak-run/src/main/java/org/apache/jackrabbit/oak/run/MetricsExporterFixtureProvider.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/MetricsExporterFixtureProvider.java (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/MetricsExporterFixtureProvider.java (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.run; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import com.codahale.metrics.MetricRegistry; +import com.google.common.base.Splitter; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.dropwizard.DropwizardExports; +import io.prometheus.client.exporter.PushGateway; +import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider; +import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.Collections.emptyMap; +import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.getService; + +/** + * Initialize different metrics exporter fixture based on parameters used. + */ +public class MetricsExporterFixtureProvider { + private static final Logger log = LoggerFactory.getLogger(NodeStoreFixtureProvider.class); + + @Nullable + public static MetricsExporterFixture create(DataStoreOptions options, Whiteboard wb) throws Exception { + if (options.exportMetrics()) { + CollectorRegistry collectorRegistry = new CollectorRegistry(); + wb.register(CollectorRegistry.class, collectorRegistry, emptyMap()); + + MetricRegistry metricRegistry = getService(wb, MetricRegistry.class); + + ExportMetricsArgs + metricsArgs = new ExportMetricsArgs(options.exportMetricsArgs()); + if (metricsArgs.getExporterType() == ExporterType.pushgateway) { + PushGateway pg = new PushGateway(metricsArgs.getPushUri()); + new DropwizardExports(metricRegistry).register(collectorRegistry); + + wb.register(PushGateway.class, pg, emptyMap()); + return new MetricsExporterFixture() { + @Override public ExporterType getExporterType() { + return ExporterType.pushgateway; + } + + @Override public PushGateway getMetricsExporter() { + return pg; + } + + @Override public void close() throws IOException { + pg.pushAdd(collectorRegistry, PushGateway.class.getName(), metricsArgs.getPushMap()); + } + }; + } + } + return null; + } + + /** + * Metrics exporter arguments + * 1. Exporter Type + * 2. Exporter URI + * 3. Metadata map + * e.g. pushgateway;uri;key1=value1,key2=value2 + */ + static class ExportMetricsArgs { + private final ExporterType exporterType; + private final String pushUri; + private final Map pushMap; + + ExportMetricsArgs(String args) { + List split = Splitter.on(";").limit(3).omitEmptyStrings().trimResults().splitToList(args); + this.exporterType = ExporterType.valueOf(split.get(0)); + + if (split.size() < 2) { + throw new IllegalArgumentException("No URL defined"); + } + + this.pushUri = split.get(1); + + if (split.size() > 2) { + this.pushMap = Splitter.on(",").omitEmptyStrings().trimResults().withKeyValueSeparator("=").split(split.get(2)); + } else { + this.pushMap = emptyMap(); + } + log.info("Map of properties pushed [{}]", pushMap); + } + + public String getPushUri() { + return pushUri; + } + + public Map getPushMap() { + return pushMap; + } + + public ExporterType getExporterType() { + return exporterType; + } + } + + + /** + * Exporter Type supported + */ + public enum ExporterType { + pushgateway("Prometheus Push Gateway"); + + private String type; + + ExporterType(String type) { + this.type = type; + } + } +} Index: oak-run/src/test/java/org/apache/jackrabbit/oak/run/MetricsExporterFixtureProviderTest.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-run/src/test/java/org/apache/jackrabbit/oak/run/MetricsExporterFixtureProviderTest.java (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) +++ oak-run/src/test/java/org/apache/jackrabbit/oak/run/MetricsExporterFixtureProviderTest.java (revision 5b3a27fd29083c243e678293e4ea7b5b00bb2de2) @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.run; + +import java.util.Map; + +import com.google.common.collect.Maps; +import io.prometheus.client.exporter.PushGateway; +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import org.apache.jackrabbit.oak.run.MetricsExporterFixtureProvider.ExportMetricsArgs; +import org.apache.jackrabbit.oak.spi.whiteboard.DefaultWhiteboard; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests for MetricsExporterFixtureProvider + */ +public class MetricsExporterFixtureProviderTest { + + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + @Test + public void checkCorrectPushGatewayInit() throws Exception { + OptionParser parser = new OptionParser(); + DataStoreOptions dataStoreOptions = new DataStoreOptions(parser); + + OptionSet option = parser.parse("--export-metrics", "pushgateway;localhost:9091;key1=value1,key2=value2"); + dataStoreOptions.configure(option); + + MetricsExporterFixture metricsExporterFixture = + MetricsExporterFixtureProvider.create(dataStoreOptions, new DefaultWhiteboard()); + + assertEquals("pushgateway", metricsExporterFixture.getExporterType().name()); + Object metricsExporter = metricsExporterFixture.getMetricsExporter(); + assertTrue(metricsExporter instanceof PushGateway); + } + + @Test + public void testMetricArgs() throws Exception { + String option = "pushgateway;localhost:9091;key1=value1,key2=value2"; + Map expectedMap = Maps.newHashMap(); + expectedMap.put("key1", "value1"); + expectedMap.put("key2", "value2"); + + ExportMetricsArgs metricsArgs = new ExportMetricsArgs(option); + + assertEquals("pushgateway", metricsArgs.getExporterType().name()); + assertEquals("localhost:9091", metricsArgs.getPushUri()); + assertEquals(expectedMap, metricsArgs.getPushMap()); + } + + @Test + public void testMetricArgsNoType() throws Exception { + expectedEx.expect(java.lang.IllegalArgumentException.class); + + String option = "localhost:9091;key1=value1,key2=value2"; + + ExportMetricsArgs metricsArgs = new ExportMetricsArgs(option); + } + + @Test + public void testMetricArgsWrongType() throws Exception { + expectedEx.expect(java.lang.IllegalArgumentException.class); + + String option = "wrongtype:localhost:9091;key1=value1,key2=value2"; + + ExportMetricsArgs metricsArgs = new ExportMetricsArgs(option); + } + + @Test + public void testMetricArgsNoProps() throws Exception { + String option = "pushgateway;localhost:9091"; + + ExportMetricsArgs metricsArgs = new ExportMetricsArgs(option); + + assertEquals("pushgateway", metricsArgs.getExporterType().name()); + assertEquals("localhost:9091", metricsArgs.getPushUri()); + assertEquals(Maps.newHashMap(), metricsArgs.getPushMap()); + } + + @Test + public void testMetricArgsNoUrlNoMap() throws Exception { + expectedEx.expect(java.lang.IllegalArgumentException.class); + + String option = "pushgateway"; + + ExportMetricsArgs metricsArgs = new ExportMetricsArgs(option); + } + + @Test + public void testMetricArgsNoUrl() throws Exception { + expectedEx.expect(java.lang.IllegalArgumentException.class); + + String option = "pushgateway:key1=value1,key2=value2"; + Map expectedMap = Maps.newHashMap(); + expectedMap.put("key1", "value1"); + expectedMap.put("key2", "value2"); + + ExportMetricsArgs metricsArgs = new ExportMetricsArgs(option); + assertEquals("pushgateway", metricsArgs.getExporterType().name()); + assertNotEquals(expectedMap, metricsArgs.getPushMap()); + assertNotEquals("localhost:9091", metricsArgs.getPushUri()); + } +}