From 3f1eb6e4fb03688fd71549597bb57840e884f3b2 Mon Sep 17 00:00:00 2001 From: stack Date: Tue, 23 Dec 2014 14:47:11 -0800 Subject: [PATCH] HTRACE-23 First cut at src release packaging Add a NOTICE.txt listing our includes (one of the go packages has no license but I filed and issue) Added a src/assembly/src.xml presecription for what the src tarball should include. Moved site under src/main/site since we've had to add a src dir. Added assembly plugin to poms, made apache our parent pom. --- NOTICE.txt | 18 +++ README.md | 2 +- htrace-core/pom.xml | 8 ++ htrace-flume/pom.xml | 8 ++ htrace-hbase/pom.xml | 8 ++ htrace-zipkin/pom.xml | 8 ++ pom.xml | 29 +++-- site/markdown/index.md | 238 ---------------------------------------- site/site.xml | 70 ------------ src/main/assembly/src.xml | 86 +++++++++++++++ src/main/site/markdown/index.md | 238 ++++++++++++++++++++++++++++++++++++++++ src/main/site/site.xml | 70 ++++++++++++ 12 files changed, 466 insertions(+), 317 deletions(-) create mode 100644 NOTICE.txt delete mode 100644 site/markdown/index.md delete mode 100644 site/site.xml create mode 100644 src/main/assembly/src.xml create mode 100644 src/main/site/markdown/index.md create mode 100644 src/main/site/site.xml diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 0000000..19f97eb --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,18 @@ +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +In addition, this product includes software developed by: + +JUnit (http://www.junit.org/) included under the Common Public License v1.0. See +the full text here: http://junit.sourceforge.net/cpl-v10.html + +levigo, a go wrapper for leveldb, is copyright Jeffrey M Hodges and +is MIT licensed: https://github.com/jmhodges/levigo/blob/master/LICENSE + +Units, unit multipliers and functions for go, has license +(TBD https://github.com/alecthomas/units/issues/1). +It is by alecthomas: https://github.com/alecthomas/units + +Kingpin, a go command line and flag parser is licensed MIT +(https://github.com/alecthomas/kingpin/blob/master/COPYING) +by alecthomas diff --git a/README.md b/README.md index e5fd84f..8d5bb3c 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,4 @@ HTrace ====== HTrace is a tracing framework for use with distributed systems written in java. -See documentation at site/markdown/index.md or at http://htrace.incubator.apache.org. +See documentation at src/main/site/markdown/index.md or at http://htrace.incubator.apache.org. diff --git a/htrace-core/pom.xml b/htrace-core/pom.xml index cf4942c..ec35841 100644 --- a/htrace-core/pom.xml +++ b/htrace-core/pom.xml @@ -19,6 +19,7 @@ language governing permissions and limitations under the License. --> htrace org.apache.htrace 3.1.0-SNAPSHOT + .. htrace-core @@ -31,6 +32,13 @@ language governing permissions and limitations under the License. --> + + maven-assembly-plugin + + true + + + org.apache.maven.plugins maven-source-plugin diff --git a/htrace-flume/pom.xml b/htrace-flume/pom.xml index 1bf258b..ee83329 100644 --- a/htrace-flume/pom.xml +++ b/htrace-flume/pom.xml @@ -19,6 +19,7 @@ language governing permissions and limitations under the License. --> htrace org.apache.htrace 3.1.0-SNAPSHOT + .. htrace-flume @@ -33,6 +34,13 @@ language governing permissions and limitations under the License. --> + + maven-assembly-plugin + + true + + + org.apache.maven.plugins maven-source-plugin diff --git a/htrace-hbase/pom.xml b/htrace-hbase/pom.xml index 2ff26aa..01d0db0 100644 --- a/htrace-hbase/pom.xml +++ b/htrace-hbase/pom.xml @@ -19,6 +19,7 @@ language governing permissions and limitations under the License. --> htrace org.apache.htrace 3.1.0-SNAPSHOT + .. htrace-hbase @@ -48,6 +49,13 @@ language governing permissions and limitations under the License. --> + + maven-assembly-plugin + + true + + + org.apache.rat apache-rat-plugin diff --git a/htrace-zipkin/pom.xml b/htrace-zipkin/pom.xml index e43fed4..3679318 100644 --- a/htrace-zipkin/pom.xml +++ b/htrace-zipkin/pom.xml @@ -19,6 +19,7 @@ language governing permissions and limitations under the License. --> htrace org.apache.htrace 3.1.0-SNAPSHOT + .. htrace-zipkin @@ -31,6 +32,13 @@ language governing permissions and limitations under the License. --> + + maven-assembly-plugin + + true + + + org.apache.maven.plugins maven-source-plugin diff --git a/pom.xml b/pom.xml index 22853f3..43fe336 100644 --- a/pom.xml +++ b/pom.xml @@ -11,6 +11,13 @@ OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> 4.0.0 + + org.apache + apache + 12 + + + org.apache.htrace htrace @@ -27,13 +34,6 @@ language governing permissions and limitations under the License. --> htrace-flume - - org.sonatype.oss - oss-parent - 7 - - - The Apache Software License, Version 2.0 @@ -250,11 +250,24 @@ language governing permissions and limitations under the License. --> - ${basedir}/site + ${basedir}/src/main/site UTF-8 UTF-8 + + maven-assembly-plugin + 2.5.3 + + + false + gnu + htrace-${project.version} + + src/main/assembly/src.xml + + + diff --git a/site/markdown/index.md b/site/markdown/index.md deleted file mode 100644 index 09b71a9..0000000 --- a/site/markdown/index.md +++ /dev/null @@ -1,238 +0,0 @@ -Apache HTrace is an Apache Incubator -project. To add HTrace to your project, see detail on how to add it as a -dependency. - -Formally, HTrace was available at org.htrace. - -API ---- -Using HTrace requires adding some instrumentation to your application. -Before we get into the details, lets review our terminology. HTrace -borrows [Dapper's](http://research.google.com/pubs/pub36356.html) -terminology. - -Span: The basic unit of work. For example, sending an RPC is a -new span, as is sending a response to an RPC. -Span's are identified by a unique 64-bit ID for the span and another -64-bit ID for the trace the span is a part of. Spans also have other -data, such as descriptions, key-value annotations, the ID of the span -that caused them, and process ID's (normally IP address). - -Spans are started and stopped, and they keep track of their timing -information. Once you create a span, you must stop it at some point -in the future. - -Trace: A set of spans forming a tree-like structure. For -example, if you are running a distributed big-data store, a trace -might be formed by a put request. - -### How to add tracing to your application -To instrument your system you must: - -1. Attach additional information to your RPC's. -In order to create the causal links necessary for a trace, HTrace -needs to know about the causal -relationships between spans. The only information you need to add to -your RPC's is two 64-bit longs. If tracing is enabled (Trace.isTracing() -returns true) when you send an RPC, attach the ID of the current span -and the ID of the current trace to the message. -On the receiving end of the RPC, check to see if the message has the -additional tracing information above. If it does, start a new span -with the information given (more on that in a bit). - -2. Wrap your thread changes. -HTrace stores span information in java's ThreadLocals, which causes -the trace to be "lost" on thread changes. The only way to prevent -this is to "wrap" your thread changes. For example, if your code looks -like this: - -````java - Thread t1 = new Thread(new MyRunnable()); - ... -```` - -Just change it to look this: - -````java - Thread t1 = new Thread(Trace.wrap(new MyRunnable())); -```` - -That's it! `Trace.wrap()` takes a single argument (a runnable or a -callable) and if the current thread is a part of a trace, returns a -wrapped version of the argument. The wrapped version of a callable -and runnable just knows about the span that created it and will start -a new span in the new thread that is the child of the span that -created the runnable/callable. There may be situations in which a -simple `Trace.wrap()` does not suffice. In these cases all you need -to do is keep a reference to the "parent span" (the span before the -thread change) and once you're in the new thread start a new span that -is the "child" of the parent span you stored. - -For example: - -Say you have some object representing a "put" operation. When the -client does a "put," the put is first added to a list so another -thread can batch together the puts. In this situation, you -might want to add another field to the Put class that could store the -current span at the time the put was created. Then when the put is -pulled out of the list to be processed, you can start a new span as -the child of the span stored in the Put. - -3. Add custom spans and annotations. -Once you've augmented your RPC's and wrapped the necessary thread -changes, you can add more spans and annotations wherever you want. -For example, you might do some expensive computation that you want to -see on your traces. In this case, you could start a new span before -the computation that you then stop after the computation has -finished. It might look like this: - -````java - Span computationSpan = Trace.startSpan("Expensive computation."); - try { - //expensive computation here - } finally { - computationSpan.stop(); - } -```` - -HTrace also supports key-value annotations on a per-trace basis. - -Example: - -````java - Trace.currentTrace().addAnnotation("faultyRecordCounter".getBytes(), "1".getBytes()); -```` - -`Trace.currentTrace()` will not return `null` if the current thread is -not tracing, but instead it will return a `NullSpan`, which does -nothing on any of its method calls. The takeaway here is you can call -methods on the `currentTrace()` without fear of NullPointerExceptions. - -###Samplers -`Sampler` is an interface that defines one function: - -````java - boolean next(T info); -```` - -All of the `Trace.startSpan()` methods can take an optional sampler. -A new span is only created if the sampler's next function returns -true. If the Sampler returns false, the `NullSpan` is returned from -`startSpan()`, so it's safe to call `stop()` or `addAnnotation()` on it. -As you may have noticed from the `next()` method signature, Sampler is -parameterized. The argument to `next()` is whatever piece of -information you might need for sampling. See `Sampler.java` for an -example of this. If you do not require any additional information, -then just ignore the parameter. -HTrace includes a sampler that always returns true, a -sampler that always returns false and a sampler returns true some -percentage of the time (you pass in the percentage as a decimal at construction). - -HTrace comes with several standard samplers, including `AlwaysSampler`, -`NeverSampler`, `ProbabilitySampler`, and `CountSampler`. An application can -use the `SamplerBuilder` to create one of these standard samplers based on the -current configuration. - -````java - HTraceConfiguration hconf = createMyHTraceConfiguration(); - Sampler sampler = new SamplerBuilder(hconf).build(); -```` - -####Trace.startSpan() -There is a single method to create and start spans: `startSpan()`. -For the `startSpan()` methods that do not take an explicit Sampler, the -default Sampler is used. The default sampler returns true if and only -if tracing is already on in the current thread. That means that -calling `startSpan()` with no explicit Sampler is a good idea when you -have information that you would like to add to a trace if it's already -occurring, but is not something you would want to start a whole new -trace for. - -If you are using a sampler that makes use of the `T info` parameter to -`next()`, just pass in the object as the last argument. If you leave it -out, HTrace will pass `null` for you (so make sure your Samplers can -handle `null`). - -Aside from whether or not you pass in an explicit `Sampler`, there are -other options you have when calling `startSpan()`. -For the next section I am assuming you are familiar with the options -for passing in `Samplers` and `info` parameters, so when I say "no -arguments," I mean no additional arguments other than whatever -`Sampler`/`info` parameters you deem necessary. - -You can call `startSpan()` with no additional arguments. -In this case, `Trace.java` will start a span if the sampler (explicit -or default) returns true. If the current span is not the `NullSpan`, the span -returned will be a child of the current span, otherwise it will start -a new trace in the current thread (it will be a -`ProcessRootMilliSpan`). All of the other `startSpan()` methods take some -parameter describing the parent span of the span to be created. The -versions that take a `TraceInfo` or a `long traceId` and `long -parentId` will mostly be used when continuing a trace over RPC. The -receiver of the RPC will check the message for the additional two -`longs` and will call `startSpan()` if they are attached. The last -`startSpan()` takes a `Span parent`. The result of `parent.child()` -will be used for the new span. `Span.child()` simply returns a span -that is a child of `this`. - -###Span Receivers -In order to use the tracing information consisting of spans, -you need an implementation of `SpanReceiver` interface which collects spans -and typically writes it to files or databases or collector services. -The `SpanReceiver` implementation must provide a `receiveSpan` method which -is called from `Trace.deliver` method. -You do not need to explicitly call `Trace.deliver` -because it is internally called by the implementation of `Span`. - -````java - public interface SpanReceiver extends Closeable { - public void receiveSpan(Span span); - } -```` - -HTrace comes with several standard span receivers, such as -`LocalFileSpanReceiver`. An application can use the `SpanReceiverBuilder` to -create a particular type of standard `SpanReceiver` based on the current -configuration. Once a SpanReceiver has been created, it should be registered -with the HTrace framework by calling `Trace.addReceiver`. - -````java - HTraceConfiguration hconf = createMyHTraceConfiguration(); - SpanReceiverBuilder builder = new SpanReceiverBuilder(hconf); - SpanReceiver spanReceiver = builder.build(); - if (spanReceiver != null) { - Trace.addReceiver(spanReceiver); - } -```` - -####Zipkin -htrace-zipkin provides the `SpanReceiver` implementation -which sends spans to [Zipkin](https://github.com/twitter/zipkin) collector. -You can build the uber-jar (htrace-zipkin-*-jar-withdependency.jar) for manual -setup as shown below. This uber-jar contains all dependencies except -htrace-core and its dependencies. - - $ cd htrace-zipkin - $ mvn compile assembly:single - -####HBase Receiver -See htrace-hbase for an Span Receiver implementation that writes HBase. -Also bundled is a simple Span Viewer. - -Testing Information -------------------------------- - -The test that creates a sample trace (TestHTrace) takes a command line -argument telling it where to write span information. Run -`mvn test -DargLine="-DspanFile=FILE\_PATH"` to write span -information to FILE_PATH. If no file is specified, span information -will be written to standard out. If span information is written to a -file, you can use the included graphDrawer python script in tools/ -to create a simple visualization of the trace. Or you could write -some javascript to make a better visualization, and send a pull -request if you do :). - -Publishing to Maven Central -------------------------------- -See [OSSRH-8896](https://issues.sonatype.org/browse/OSSRH-8896) -for repository vitals. diff --git a/site/site.xml b/site/site.xml deleted file mode 100644 index 0c22c17..0000000 --- a/site/site.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - - - Apache HTrace - http://htrace.incubator.apache.org/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - bootswatch-flatly - top - true - A tracing framework for use with distributed systems written in java - - Home - - - - - lt.velykis.maven.skins - reflow-maven-skin - 1.1.1 - - diff --git a/src/main/assembly/src.xml b/src/main/assembly/src.xml new file mode 100644 index 0000000..d70d87f --- /dev/null +++ b/src/main/assembly/src.xml @@ -0,0 +1,86 @@ + + + + + + src + + tar.gz + + + + + true + + + + + + target/ + test/ + .classpath + .project + .git + .gitignore + .settings/ + + + + + + + + + ${project.basedir}/src + src + 0644 + 0755 + + + + ${project.basedir}/bin + bin + 0755 + 0755 + + + ${project.basedir}/src + src + + + ${project.basedir}/tools + tools + + + ${project.basedir} + . + + pom.xml + LICENSE.txt + NOTICE.txt + CHANGES.txt + README.md + + 0644 + + + diff --git a/src/main/site/markdown/index.md b/src/main/site/markdown/index.md new file mode 100644 index 0000000..09b71a9 --- /dev/null +++ b/src/main/site/markdown/index.md @@ -0,0 +1,238 @@ +Apache HTrace is an Apache Incubator +project. To add HTrace to your project, see detail on how to add it as a +dependency. + +Formally, HTrace was available at org.htrace. + +API +--- +Using HTrace requires adding some instrumentation to your application. +Before we get into the details, lets review our terminology. HTrace +borrows [Dapper's](http://research.google.com/pubs/pub36356.html) +terminology. + +Span: The basic unit of work. For example, sending an RPC is a +new span, as is sending a response to an RPC. +Span's are identified by a unique 64-bit ID for the span and another +64-bit ID for the trace the span is a part of. Spans also have other +data, such as descriptions, key-value annotations, the ID of the span +that caused them, and process ID's (normally IP address). + +Spans are started and stopped, and they keep track of their timing +information. Once you create a span, you must stop it at some point +in the future. + +Trace: A set of spans forming a tree-like structure. For +example, if you are running a distributed big-data store, a trace +might be formed by a put request. + +### How to add tracing to your application +To instrument your system you must: + +1. Attach additional information to your RPC's. +In order to create the causal links necessary for a trace, HTrace +needs to know about the causal +relationships between spans. The only information you need to add to +your RPC's is two 64-bit longs. If tracing is enabled (Trace.isTracing() +returns true) when you send an RPC, attach the ID of the current span +and the ID of the current trace to the message. +On the receiving end of the RPC, check to see if the message has the +additional tracing information above. If it does, start a new span +with the information given (more on that in a bit). + +2. Wrap your thread changes. +HTrace stores span information in java's ThreadLocals, which causes +the trace to be "lost" on thread changes. The only way to prevent +this is to "wrap" your thread changes. For example, if your code looks +like this: + +````java + Thread t1 = new Thread(new MyRunnable()); + ... +```` + +Just change it to look this: + +````java + Thread t1 = new Thread(Trace.wrap(new MyRunnable())); +```` + +That's it! `Trace.wrap()` takes a single argument (a runnable or a +callable) and if the current thread is a part of a trace, returns a +wrapped version of the argument. The wrapped version of a callable +and runnable just knows about the span that created it and will start +a new span in the new thread that is the child of the span that +created the runnable/callable. There may be situations in which a +simple `Trace.wrap()` does not suffice. In these cases all you need +to do is keep a reference to the "parent span" (the span before the +thread change) and once you're in the new thread start a new span that +is the "child" of the parent span you stored. + +For example: + +Say you have some object representing a "put" operation. When the +client does a "put," the put is first added to a list so another +thread can batch together the puts. In this situation, you +might want to add another field to the Put class that could store the +current span at the time the put was created. Then when the put is +pulled out of the list to be processed, you can start a new span as +the child of the span stored in the Put. + +3. Add custom spans and annotations. +Once you've augmented your RPC's and wrapped the necessary thread +changes, you can add more spans and annotations wherever you want. +For example, you might do some expensive computation that you want to +see on your traces. In this case, you could start a new span before +the computation that you then stop after the computation has +finished. It might look like this: + +````java + Span computationSpan = Trace.startSpan("Expensive computation."); + try { + //expensive computation here + } finally { + computationSpan.stop(); + } +```` + +HTrace also supports key-value annotations on a per-trace basis. + +Example: + +````java + Trace.currentTrace().addAnnotation("faultyRecordCounter".getBytes(), "1".getBytes()); +```` + +`Trace.currentTrace()` will not return `null` if the current thread is +not tracing, but instead it will return a `NullSpan`, which does +nothing on any of its method calls. The takeaway here is you can call +methods on the `currentTrace()` without fear of NullPointerExceptions. + +###Samplers +`Sampler` is an interface that defines one function: + +````java + boolean next(T info); +```` + +All of the `Trace.startSpan()` methods can take an optional sampler. +A new span is only created if the sampler's next function returns +true. If the Sampler returns false, the `NullSpan` is returned from +`startSpan()`, so it's safe to call `stop()` or `addAnnotation()` on it. +As you may have noticed from the `next()` method signature, Sampler is +parameterized. The argument to `next()` is whatever piece of +information you might need for sampling. See `Sampler.java` for an +example of this. If you do not require any additional information, +then just ignore the parameter. +HTrace includes a sampler that always returns true, a +sampler that always returns false and a sampler returns true some +percentage of the time (you pass in the percentage as a decimal at construction). + +HTrace comes with several standard samplers, including `AlwaysSampler`, +`NeverSampler`, `ProbabilitySampler`, and `CountSampler`. An application can +use the `SamplerBuilder` to create one of these standard samplers based on the +current configuration. + +````java + HTraceConfiguration hconf = createMyHTraceConfiguration(); + Sampler sampler = new SamplerBuilder(hconf).build(); +```` + +####Trace.startSpan() +There is a single method to create and start spans: `startSpan()`. +For the `startSpan()` methods that do not take an explicit Sampler, the +default Sampler is used. The default sampler returns true if and only +if tracing is already on in the current thread. That means that +calling `startSpan()` with no explicit Sampler is a good idea when you +have information that you would like to add to a trace if it's already +occurring, but is not something you would want to start a whole new +trace for. + +If you are using a sampler that makes use of the `T info` parameter to +`next()`, just pass in the object as the last argument. If you leave it +out, HTrace will pass `null` for you (so make sure your Samplers can +handle `null`). + +Aside from whether or not you pass in an explicit `Sampler`, there are +other options you have when calling `startSpan()`. +For the next section I am assuming you are familiar with the options +for passing in `Samplers` and `info` parameters, so when I say "no +arguments," I mean no additional arguments other than whatever +`Sampler`/`info` parameters you deem necessary. + +You can call `startSpan()` with no additional arguments. +In this case, `Trace.java` will start a span if the sampler (explicit +or default) returns true. If the current span is not the `NullSpan`, the span +returned will be a child of the current span, otherwise it will start +a new trace in the current thread (it will be a +`ProcessRootMilliSpan`). All of the other `startSpan()` methods take some +parameter describing the parent span of the span to be created. The +versions that take a `TraceInfo` or a `long traceId` and `long +parentId` will mostly be used when continuing a trace over RPC. The +receiver of the RPC will check the message for the additional two +`longs` and will call `startSpan()` if they are attached. The last +`startSpan()` takes a `Span parent`. The result of `parent.child()` +will be used for the new span. `Span.child()` simply returns a span +that is a child of `this`. + +###Span Receivers +In order to use the tracing information consisting of spans, +you need an implementation of `SpanReceiver` interface which collects spans +and typically writes it to files or databases or collector services. +The `SpanReceiver` implementation must provide a `receiveSpan` method which +is called from `Trace.deliver` method. +You do not need to explicitly call `Trace.deliver` +because it is internally called by the implementation of `Span`. + +````java + public interface SpanReceiver extends Closeable { + public void receiveSpan(Span span); + } +```` + +HTrace comes with several standard span receivers, such as +`LocalFileSpanReceiver`. An application can use the `SpanReceiverBuilder` to +create a particular type of standard `SpanReceiver` based on the current +configuration. Once a SpanReceiver has been created, it should be registered +with the HTrace framework by calling `Trace.addReceiver`. + +````java + HTraceConfiguration hconf = createMyHTraceConfiguration(); + SpanReceiverBuilder builder = new SpanReceiverBuilder(hconf); + SpanReceiver spanReceiver = builder.build(); + if (spanReceiver != null) { + Trace.addReceiver(spanReceiver); + } +```` + +####Zipkin +htrace-zipkin provides the `SpanReceiver` implementation +which sends spans to [Zipkin](https://github.com/twitter/zipkin) collector. +You can build the uber-jar (htrace-zipkin-*-jar-withdependency.jar) for manual +setup as shown below. This uber-jar contains all dependencies except +htrace-core and its dependencies. + + $ cd htrace-zipkin + $ mvn compile assembly:single + +####HBase Receiver +See htrace-hbase for an Span Receiver implementation that writes HBase. +Also bundled is a simple Span Viewer. + +Testing Information +------------------------------- + +The test that creates a sample trace (TestHTrace) takes a command line +argument telling it where to write span information. Run +`mvn test -DargLine="-DspanFile=FILE\_PATH"` to write span +information to FILE_PATH. If no file is specified, span information +will be written to standard out. If span information is written to a +file, you can use the included graphDrawer python script in tools/ +to create a simple visualization of the trace. Or you could write +some javascript to make a better visualization, and send a pull +request if you do :). + +Publishing to Maven Central +------------------------------- +See [OSSRH-8896](https://issues.sonatype.org/browse/OSSRH-8896) +for repository vitals. diff --git a/src/main/site/site.xml b/src/main/site/site.xml new file mode 100644 index 0000000..0c22c17 --- /dev/null +++ b/src/main/site/site.xml @@ -0,0 +1,70 @@ + + + + + + + Apache HTrace + http://htrace.incubator.apache.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + bootswatch-flatly + top + true + A tracing framework for use with distributed systems written in java + + Home + + + + + lt.velykis.maven.skins + reflow-maven-skin + 1.1.1 + + -- 2.2.0