commit fcf96ee4d0a368c8dcee488c823117e090ff1beb Author: stack Date: Sat Jun 13 15:24:20 2015 -0700 HTRACE-183 Have go build into the target dir rather than under src diff --git a/htrace-htraced/go/Godeps/Godeps.json b/htrace-htraced/go/Godeps/Godeps.json new file mode 100644 index 0000000..47aa90e --- /dev/null +++ b/htrace-htraced/go/Godeps/Godeps.json @@ -0,0 +1,30 @@ +{ + "ImportPath": "git-wip-us.apache.org/repos/asf/incubator-htrace.git", + "GoVersion": "go1.3.1", + "Deps": [ + { + "ImportPath": "github.com/alecthomas/kingpin", + "Rev": "afafa8aab106d31c9dc8f5e562b3f30f6246c3d4" + }, + { + "ImportPath": "github.com/alecthomas/units", + "Rev": "6b4e7dc5e3143b85ea77909c72caf89416fc2915" + }, + { + "ImportPath": "github.com/gorilla/context", + "Rev": "215affda49addc4c8ef7e2534915df2c8c35c6cd" + }, + { + "ImportPath": "github.com/gorilla/mux", + "Rev": "e444e69cbd2e2e3e0749a2f3c717cec491552bbf" + }, + { + "ImportPath": "github.com/jmhodges/levigo", + "Rev": "2c43dde93d0e056173706534afd514fcbc1dd578" + }, + { + "ImportPath": "github.com/ugorji/go/codec", + "Rev": "08bbe4aa39b9f189f4e294b5c8408b5fa5787bb2" + } + ] +} diff --git a/htrace-htraced/go/format.sh b/htrace-htraced/go/format.sh new file mode 100755 index 0000000..46aa5b1 --- /dev/null +++ b/htrace-htraced/go/format.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Reformats the HTrace code. +# +# ./format.sh Reformats all code. +# + +die() { + echo $@ + exit 1 +} + +# Check for gofmt. It should be installed whenever the go developement tools +# are installed. +which gofmt &> /dev/null +[ $? -ne 0 ] && die "You must install the gofmt code reformatting formatting tool." + +# Find go sources. We assume no newlines or whitespace in file names. +SCRIPT_DIR="$(cd "$( dirname $0 )" && pwd)" +find "${SCRIPT_DIR}/src" -noleaf -xdev -name '*.go' | xargs -L 1 gofmt -w diff --git a/htrace-htraced/go/gobuild.sh b/htrace-htraced/go/gobuild.sh new file mode 100755 index 0000000..81c9f7d --- /dev/null +++ b/htrace-htraced/go/gobuild.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Builds the HTrace server code. +# +# ./build.sh Builds the code. +# ./build.sh test Builds and runs all unit tests. +# ./build.sh bench Builds and runs all benchmarks +# + +die() { + echo $@ + exit 1 +} + +ACTION=install +if [ $# -gt 0 ]; then + ACTION="${1}" + shift +fi +RELEASE_VERSION=${RELEASE_VERSION:-unknown} + +# Set up directories. The build/ directory is where build dependencies and +# build binaries should go. +SCRIPT_DIR="$(cd "$( dirname $0 )" && pwd)" +export GOBIN="${SCRIPT_DIR}/build" +mkdir -p "${GOBIN}" || die "failed to mkdir -p ${GOBIN}" +cd "${GOBIN}" || die "failed to cd to ${SCRIPT_DIR}" +export GOPATH="${GOBIN}:${SCRIPT_DIR}" + +# Use the unsafe package when possible to get greater speed. For example, +# go-codec can bypass the overhead of converting between []byte and string in +# some cases when using unsafe. +TAGS="-tags unsafe" + +# Check for go +which go &> /dev/null +if [ $? -ne 0 ]; then + cat < /dev/null + [ $? -eq 0 ] && ldconfig=ldconfig +fi +if [ -n "${ldconfig}" ]; then + if "${ldconfig}" -p | grep -q libleveldb; then + : + else + echo "You must install the leveldb-devel package (or distro-specific equivalent.)" + exit 1 + fi +fi + +case $ACTION in +clean) + rm -rf -- "${GOBIN}" ${SCRIPT_DIR}/pkg + ;; +install) + # Ensure that we have the godep program. + PATH="${PATH}:${GOBIN}" + which godep &> /dev/null + if [ $? -ne 0 ]; then + echo "Installing godep..." + go get github.com/tools/godep || die "failed to get godep" + fi + + # Download dependencies into the build directory. + echo "godep restore..." + godep restore || die "failed to set up dependencies" + + # Discover the git version + GIT_VERSION=$(git rev-parse HEAD) + [ $? -eq 0 ] || GIT_VERSION="unknown" + + # Inject the release and git version into the htraced ldflags. + FLAGS="-X main.RELEASE_VERSION ${RELEASE_VERSION} -X main.GIT_VERSION ${GIT_VERSION}" + go install ${TAGS} -ldflags "${FLAGS}" -v org/apache/htrace/... "$@" + ;; +bench) + go test org/apache/htrace/... ${TAGS} -test.bench=. "$@" + ;; +*) + go ${ACTION} org/apache/htrace/... ${TAGS} "$@" + ;; +esac diff --git a/htrace-htraced/go/src/org/apache/htrace/client/client.go b/htrace-htraced/go/src/org/apache/htrace/client/client.go new file mode 100644 index 0000000..6a62e81 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/client/client.go @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package client + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "org/apache/htrace/common" + "org/apache/htrace/conf" +) + +// A golang client for htraced. +// TODO: fancier APIs for streaming spans in the background, optimize TCP stuff + +func NewClient(cnf *conf.Config) (*Client, error) { + hcl := Client{} + hcl.restAddr = cnf.Get(conf.HTRACE_WEB_ADDRESS) + hcl.hrpcAddr = cnf.Get(conf.HTRACE_HRPC_ADDRESS) + return &hcl, nil +} + +type Client struct { + // REST address of the htraced server. + restAddr string + + // HRPC address of the htraced server. + hrpcAddr string + + // The HRPC client, or null if it is not enabled. + hcr *hClient +} + +// Get the htraced server information. +func (hcl *Client) GetServerInfo() (*common.ServerInfo, error) { + buf, _, err := hcl.makeGetRequest("server/info") + if err != nil { + return nil, err + } + var info common.ServerInfo + err = json.Unmarshal(buf, &info) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ + "body %s: %s", string(buf), err.Error())) + } + return &info, nil +} + +// Get information about a trace span. Returns nil, nil if the span was not found. +func (hcl *Client) FindSpan(sid common.SpanId) (*common.Span, error) { + buf, rc, err := hcl.makeGetRequest(fmt.Sprintf("span/%016x", uint64(sid))) + if err != nil { + if rc == http.StatusNoContent { + return nil, nil + } + return nil, err + } + var span common.Span + err = json.Unmarshal(buf, &span) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error unmarshalling response "+ + "body %s: %s", string(buf), err.Error())) + } + return &span, nil +} + +func (hcl *Client) WriteSpans(req *common.WriteSpansReq) error { + if hcl.hrpcAddr == "" { + return hcl.writeSpansHttp(req) + } + hcr, err := newHClient(hcl.hrpcAddr) + if err != nil { + return err + } + defer hcr.Close() + return hcr.writeSpans(req) +} + +func (hcl *Client) writeSpansHttp(req *common.WriteSpansReq) error { + var w bytes.Buffer + var err error + for i := range req.Spans { + var buf []byte + buf, err = json.Marshal(req.Spans[i]) + if err != nil { + return errors.New(fmt.Sprintf("Error serializing span: %s", + err.Error())) + } + _, err = w.Write(buf) + if err != nil { + return errors.New(fmt.Sprintf("Error writing span: %s", + err.Error())) + } + _, err = w.Write([]byte{'\n'}) + //err = io.WriteString(&w, "\n") + if err != nil { + return errors.New(fmt.Sprintf("Error writing: %s", + err.Error())) + } + } + customHeaders := make(map[string]string) + if req.DefaultPid != "" { + customHeaders["htrace-pid"] = req.DefaultPid + } + _, _, err = hcl.makeRestRequest("POST", "writeSpans", + &w, customHeaders) + if err != nil { + return err + } + return nil +} + +// Find the child IDs of a given span ID. +func (hcl *Client) FindChildren(sid common.SpanId, lim int) ([]common.SpanId, error) { + buf, _, err := hcl.makeGetRequest(fmt.Sprintf("span/%016x/children?lim=%d", + uint64(sid), lim)) + if err != nil { + return nil, err + } + var spanIds []common.SpanId + err = json.Unmarshal(buf, &spanIds) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ + "body %s: %s", string(buf), err.Error())) + } + return spanIds, nil +} + +// Make a query +func (hcl *Client) Query(query *common.Query) ([]common.Span, error) { + in, err := json.Marshal(query) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error marshalling query: %s", err.Error())) + } + var out []byte + var url = fmt.Sprintf("query?query=%s", in) + out, _, err = hcl.makeGetRequest(url) + if err != nil { + return nil, err + } + var spans []common.Span + err = json.Unmarshal(out, &spans) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error unmarshalling results: %s", err.Error())) + } + return spans, nil +} + +var EMPTY = make(map[string]string) + +func (hcl *Client) makeGetRequest(reqName string) ([]byte, int, error) { + return hcl.makeRestRequest("GET", reqName, nil, EMPTY) +} + +// Make a general JSON REST request. +// Returns the request body, the response code, and the error. +// Note: if the response code is non-zero, the error will also be non-zero. +func (hcl *Client) makeRestRequest(reqType string, reqName string, reqBody io.Reader, + customHeaders map[string]string) ([]byte, int, error) { + url := fmt.Sprintf("http://%s/%s", + hcl.restAddr, reqName) + req, err := http.NewRequest(reqType, url, reqBody) + req.Header.Set("Content-Type", "application/json") + for k, v := range customHeaders { + req.Header.Set(k, v) + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, -1, errors.New(fmt.Sprintf("Error: error making http request to %s: %s\n", url, + err.Error())) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode, + errors.New(fmt.Sprintf("Error: got bad response status from %s: %s\n", url, resp.Status)) + } + var body []byte + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + return nil, -1, errors.New(fmt.Sprintf("Error: error reading response body: %s\n", err.Error())) + } + return body, 0, nil +} + +// Dump all spans from the htraced daemon. +func (hcl *Client) DumpAll(lim int, out chan *common.Span) error { + defer func() { + close(out) + }() + searchId := common.SpanId(0) + for { + q := common.Query{ + Lim: lim, + Predicates: []common.Predicate{ + common.Predicate{ + Op: "ge", + Field: "spanid", + Val: searchId.String(), + }, + }, + } + spans, err := hcl.Query(&q) + if err != nil { + return errors.New(fmt.Sprintf("Error querying spans with IDs at or after "+ + "%s: %s", searchId.String(), err.Error())) + } + if len(spans) == 0 { + return nil + } + for i := range spans { + out <- &spans[i] + } + searchId = spans[len(spans)-1].Id + 1 + } +} + +func (hcl *Client) Close() { + if hcl.hcr != nil { + hcl.hcr.Close() + } + hcl.restAddr = "" + hcl.hcr = nil +} diff --git a/htrace-htraced/go/src/org/apache/htrace/client/hclient.go b/htrace-htraced/go/src/org/apache/htrace/client/hclient.go new file mode 100644 index 0000000..608dd59 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/client/hclient.go @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package client + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "github.com/ugorji/go/codec" + "io" + "net" + "net/rpc" + "org/apache/htrace/common" +) + +type hClient struct { + rpcClient *rpc.Client +} + +type HrpcClientCodec struct { + rwc io.ReadWriteCloser + length uint32 +} + +func (cdc *HrpcClientCodec) WriteRequest(req *rpc.Request, msg interface{}) error { + methodId := common.HrpcMethodNameToId(req.ServiceMethod) + if methodId == common.METHOD_ID_NONE { + return errors.New(fmt.Sprintf("HrpcClientCodec: Unknown method name %s", + req.ServiceMethod)) + } + mh := new(codec.MsgpackHandle) + mh.WriteExt = true + w := bytes.NewBuffer(make([]byte, 0, 2048)) + enc := codec.NewEncoder(w, mh) + err := enc.Encode(msg) + if err != nil { + return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+ + "message as msgpack: %s", err.Error())) + } + buf := w.Bytes() + if len(buf) > common.MAX_HRPC_BODY_LENGTH { + return errors.New(fmt.Sprintf("HrpcClientCodec: message body is %d "+ + "bytes, but the maximum message size is %d bytes.", + len(buf), common.MAX_HRPC_BODY_LENGTH)) + } + hdr := common.HrpcRequestHeader{ + Magic: common.HRPC_MAGIC, + MethodId: methodId, + Seq: req.Seq, + Length: uint32(len(buf)), + } + err = binary.Write(cdc.rwc, binary.LittleEndian, &hdr) + if err != nil { + return errors.New(fmt.Sprintf("Error writing header bytes: %s", + err.Error())) + } + _, err = cdc.rwc.Write(buf) + if err != nil { + return errors.New(fmt.Sprintf("Error writing body bytes: %s", + err.Error())) + } + return nil +} + +func (cdc *HrpcClientCodec) ReadResponseHeader(resp *rpc.Response) error { + hdr := common.HrpcResponseHeader{} + err := binary.Read(cdc.rwc, binary.LittleEndian, &hdr) + if err != nil { + return errors.New(fmt.Sprintf("Error reading response header "+ + "bytes: %s", err.Error())) + } + resp.ServiceMethod = common.HrpcMethodIdToMethodName(hdr.MethodId) + if resp.ServiceMethod == "" { + return errors.New(fmt.Sprintf("Error reading response header: "+ + "invalid method ID %d.", hdr.MethodId)) + } + resp.Seq = hdr.Seq + if hdr.ErrLength > 0 { + if hdr.ErrLength > common.MAX_HRPC_ERROR_LENGTH { + return errors.New(fmt.Sprintf("Error reading response header: "+ + "error message was %d bytes long, but "+ + "MAX_HRPC_ERROR_LENGTH is %d.", + hdr.ErrLength, common.MAX_HRPC_ERROR_LENGTH)) + } + buf := make([]byte, hdr.ErrLength) + var nread int + nread, err = cdc.rwc.Read(buf) + if uint32(nread) != hdr.ErrLength { + return errors.New(fmt.Sprintf("Error reading response header: "+ + "failed to read %d bytes of error message.", nread)) + } + if err != nil { + return errors.New(fmt.Sprintf("Error reading response header: "+ + "failed to read %d bytes of error message: %s", + nread, err.Error())) + } + resp.Error = string(buf) + } else { + resp.Error = "" + } + cdc.length = hdr.Length + return nil +} + +func (cdc *HrpcClientCodec) ReadResponseBody(body interface{}) error { + mh := new(codec.MsgpackHandle) + mh.WriteExt = true + dec := codec.NewDecoder(io.LimitReader(cdc.rwc, int64(cdc.length)), mh) + err := dec.Decode(body) + if err != nil { + return errors.New(fmt.Sprintf("Failed to read response body: %s", + err.Error())) + } + return nil +} + +func (cdc *HrpcClientCodec) Close() error { + return cdc.rwc.Close() +} + +func newHClient(hrpcAddr string) (*hClient, error) { + hcr := hClient{} + conn, err := net.Dial("tcp", hrpcAddr) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error contacting the HRPC server "+ + "at %s: %s", hrpcAddr, err.Error())) + } + hcr.rpcClient = rpc.NewClientWithCodec(&HrpcClientCodec{rwc: conn}) + return &hcr, nil +} + +func (hcr *hClient) writeSpans(req *common.WriteSpansReq) error { + resp := common.WriteSpansResp{} + return hcr.rpcClient.Call(common.METHOD_NAME_WRITE_SPANS, req, &resp) +} + +func (hcr *hClient) Close() { + hcr.rpcClient.Close() +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/log.go b/htrace-htraced/go/src/org/apache/htrace/common/log.go new file mode 100644 index 0000000..305ecf3 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/log.go @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "errors" + "fmt" + "org/apache/htrace/conf" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// A logSink is a place logs can be written to. +type logSink struct { + path logPath + file *os.File + lock sync.Mutex + refCount int // protected by logFilesLock +} + +// Write to the logSink. +func (sink *logSink) write(str string) { + sink.lock.Lock() + defer sink.lock.Unlock() + _, err := sink.file.Write([]byte(str)) + if err != nil { + fmt.Fprintf(os.Stderr, "Error logging to '%s': %s\n", sink.path, err.Error()) + } +} + +// Unreference the logSink. If there are no more references, and the logSink is +// closeable, then we will close it here. +func (sink *logSink) Unref() { + logFilesLock.Lock() + defer logFilesLock.Unlock() + sink.refCount-- + if sink.refCount <= 0 { + if sink.path.IsCloseable() { + err := sink.file.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "Error closing log file %s: %s\n", + sink.path, err.Error()) + } + } + logSinks[sink.path] = nil + } +} + +type logPath string + +// An empty LogPath represents "stdout." +const STDOUT_LOG_PATH = "" + +// Convert a path to a logPath. +func logPathFromString(path string) logPath { + if path == STDOUT_LOG_PATH { + return logPath("") + } + absPath, err := filepath.Abs(path) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get absolute path of %s: %s\n", + path, err.Error()) + return logPath(path) + } + return logPath(absPath) +} + +// Convert the path to a human-readable string. +func (path logPath) String() string { + if path == "" { + return "(stdout)" + } else { + return string(path) + } +} + +// Return true if the path is closeable. stdout is not closeable. +func (path logPath) IsCloseable() bool { + return path != STDOUT_LOG_PATH +} + +func (path logPath) Open() *logSink { + if path == STDOUT_LOG_PATH { + return &logSink{path: path, file: os.Stdout} + } + file, err := os.OpenFile(string(path), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + sink := &logSink{path: STDOUT_LOG_PATH, file: os.Stdout} + fmt.Fprintf(os.Stderr, "Failed to open log file %s: %s\n", + path, err.Error()) + return sink + } + return &logSink{path: path, file: file} +} + +var logFilesLock sync.Mutex + +var logSinks map[logPath]*logSink = make(map[logPath]*logSink) + +func getOrCreateLogSink(pathStr string) *logSink { + path := logPathFromString(pathStr) + logFilesLock.Lock() + defer logFilesLock.Unlock() + sink := logSinks[path] + if sink == nil { + sink = path.Open() + logSinks[path] = sink + } + sink.refCount++ + return sink +} + +type Level int + +const ( + TRACE Level = iota + DEBUG + INFO + WARN + ERROR +) + +var levelToString map[Level]string = map[Level]string{ + TRACE: "TRACE", + DEBUG: "DEBUG", + INFO: "INFO", + WARN: "WARN", + ERROR: "ERROR", +} + +func (level Level) String() string { + return levelToString[level] +} + +func (level Level) LogString() string { + return level.String()[0:1] +} + +func LevelFromString(str string) (Level, error) { + for k, v := range levelToString { + if strings.ToLower(v) == strings.ToLower(str) { + return k, nil + } + } + var levelNames sort.StringSlice + levelNames = make([]string, len(levelToString)) + var i int + for _, v := range levelToString { + levelNames[i] = v + i++ + } + sort.Sort(levelNames) + return TRACE, errors.New(fmt.Sprintf("No such level as '%s'. Valid "+ + "levels are '%v'\n", str, levelNames)) +} + +type Logger struct { + sink *logSink + Level Level +} + +func NewLogger(faculty string, cnf *conf.Config) *Logger { + path, level := parseConf(faculty, cnf) + sink := getOrCreateLogSink(path) + return &Logger{sink: sink, Level: level} +} + +func parseConf(faculty string, cnf *conf.Config) (string, Level) { + facultyLogPathKey := faculty + "." + conf.HTRACE_LOG_PATH + var facultyLogPath string + if cnf.Contains(facultyLogPathKey) { + facultyLogPath = cnf.Get(facultyLogPathKey) + } else { + facultyLogPath = cnf.Get(conf.HTRACE_LOG_PATH) + } + facultyLogLevelKey := faculty + conf.HTRACE_LOG_LEVEL + var facultyLogLevelStr string + if cnf.Contains(facultyLogLevelKey) { + facultyLogLevelStr = cnf.Get(facultyLogLevelKey) + } else { + facultyLogLevelStr = cnf.Get(conf.HTRACE_LOG_LEVEL) + } + level, err := LevelFromString(facultyLogLevelStr) + if err != nil { + fmt.Fprintf(os.Stderr, "Error configuring log level: %s. Using TRACE.\n") + level = TRACE + } + return facultyLogPath, level +} + +func (lg *Logger) Trace(str string) { + lg.Write(TRACE, str) +} + +func (lg *Logger) Tracef(format string, v ...interface{}) { + lg.Write(TRACE, fmt.Sprintf(format, v...)) +} + +func (lg *Logger) Debug(str string) { + lg.Write(DEBUG, str) +} + +func (lg *Logger) Debugf(format string, v ...interface{}) { + lg.Write(DEBUG, fmt.Sprintf(format, v...)) +} + +func (lg *Logger) Info(str string) { + lg.Write(INFO, str) +} + +func (lg *Logger) Infof(format string, v ...interface{}) { + lg.Write(INFO, fmt.Sprintf(format, v...)) +} + +func (lg *Logger) Warn(str string) error { + lg.Write(WARN, str) + return errors.New(str) +} + +func (lg *Logger) Warnf(format string, v ...interface{}) error { + str := fmt.Sprintf(format, v...) + lg.Write(WARN, str) + return errors.New(str) +} + +func (lg *Logger) Error(str string) error { + lg.Write(ERROR, str) + return errors.New(str) +} + +func (lg *Logger) Errorf(format string, v ...interface{}) error { + str := fmt.Sprintf(format, v...) + lg.Write(ERROR, str) + return errors.New(str) +} + +func (lg *Logger) Write(level Level, str string) { + if level >= lg.Level { + lg.sink.write(time.Now().Format(time.RFC3339) + " " + + level.LogString() + ": " + str) + } +} + +// +// A few functions which can be used to determine if a certain level of tracing +// is enabled. These are useful in situations when evaluating the parameters +// of a logging function is expensive. (Note, however, that we don't pay the +// cost of string concatenation and manipulation when a log message doesn't +// trigger.) +// + +func (lg *Logger) TraceEnabled() bool { + return lg.Level >= TRACE +} + +func (lg *Logger) DebugEnabled() bool { + return lg.Level >= DEBUG +} + +func (lg *Logger) InfoEnabled() bool { + return lg.Level >= INFO +} + +func (lg *Logger) WarnEnabled() bool { + return lg.Level >= WARN +} + +func (lg *Logger) ErrorEnabled() bool { + return lg.Level >= ERROR +} + +func (lg *Logger) Close() { + lg.sink.Unref() + lg.sink = nil +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/log_test.go b/htrace-htraced/go/src/org/apache/htrace/common/log_test.go new file mode 100644 index 0000000..b415ce2 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/log_test.go @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "org/apache/htrace/conf" + "os" + "strings" + "testing" +) + +func newLogger(faculty string, args ...string) *Logger { + cnfBld := conf.Builder{Defaults: conf.DEFAULTS} + cnf, err := cnfBld.Build() + if err != nil { + panic(fmt.Sprintf("failed to create conf: %s", err.Error())) + } + cnf2 := cnf.Clone(args...) + lg := NewLogger(faculty, cnf2) + return lg +} + +func TestNewLogger(t *testing.T) { + lg := newLogger("foo", "log.level", "TRACE") + lg.Close() +} + +func verifyLines(t *testing.T, rdr io.Reader, lines []string) { + scanner := bufio.NewScanner(rdr) + lineIdx := 0 + for scanner.Scan() { + line := scanner.Text() + if !strings.Contains(line, lines[lineIdx]) { + t.Fatalf("Error on line %d: didn't find substring '%s' in line '%s'\n", + (lineIdx + 1), lines[lineIdx], line) + } + lineIdx++ + } + if err := scanner.Err(); err != nil { + t.Fatal(err.Error()) + } +} + +func TestFileLogs(t *testing.T) { + tempDir, err := ioutil.TempDir(os.TempDir(), "testFileLogs") + if err != nil { + panic(fmt.Sprintf("error creating tempdir: %s\n", err.Error())) + } + defer os.RemoveAll(tempDir) + logPath := tempDir + conf.PATH_SEP + "log" + lg := newLogger("foo", "log.level", "DEBUG", + "foo.log.level", "INFO", + "log.path", logPath) + lg.Tracef("Non-important stuff, ignore this.\n") + lg.Infof("problem with the foobar\n") + lg.Tracef("More non-important stuff, also ignore this.\n") + lg.Infof("and another problem with the foobar\n") + logFile, err := os.Open(logPath) + if err != nil { + t.Fatalf("failed to open file %s: %s\n", logPath, err.Error()) + } + verifyLines(t, logFile, []string{ + "problem with the foobar", + "and another problem with the foobar", + }) + logFile.Close() + lg.Close() +} + +func TestMultipleFileLogs(t *testing.T) { + tempDir, err := ioutil.TempDir(os.TempDir(), "testMultipleFileLogs") + if err != nil { + panic(fmt.Sprintf("error creating tempdir: %s\n", err.Error())) + } + defer os.RemoveAll(tempDir) + logPath := tempDir + conf.PATH_SEP + "log" + fooLg := newLogger("foo", "log.level", "DEBUG", + "foo.log.level", "INFO", + "log.path", logPath) + fooLg.Infof("The foo needs maintenance.\n") + barLg := newLogger("bar", "log.level", "DEBUG", + "foo.log.level", "INFO", + "log.path", logPath) + barLg.Debugf("The bar is open\n") + fooLg.Errorf("Fizz buzz\n") + logFile, err := os.Open(logPath) + if err != nil { + t.Fatalf("failed to open file %s: %s\n", logPath, err.Error()) + } + fooLg.Tracef("Fizz buzz2\n") + barLg.Tracef("Fizz buzz3\n") + verifyLines(t, logFile, []string{ + "The foo needs maintenance.", + "The bar is open", + "Fizz buzz", + "Fizz buzz3", + }) + logFile.Close() + fooLg.Close() + barLg.Close() +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/process.go b/htrace-htraced/go/src/org/apache/htrace/common/process.go new file mode 100644 index 0000000..d138178 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/process.go @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "bufio" + "org/apache/htrace/conf" + "os" + "os/signal" + "syscall" +) + +func LoadApplicationConfig() *conf.Config { + cnf, dlog := conf.LoadApplicationConfig() + lg := NewLogger("conf", cnf) + defer lg.Close() + if lg.Level <= DEBUG { + // Print out the debug information from loading the configuration. + scanner := bufio.NewScanner(dlog) + for scanner.Scan() { + lg.Debugf(scanner.Text() + "\n") + } + } + return cnf +} + +func InstallSignalHandlers(cnf *conf.Config) { + fatalSigs := []os.Signal{ + os.Interrupt, + os.Kill, + syscall.SIGINT, + syscall.SIGABRT, + syscall.SIGALRM, + syscall.SIGBUS, + syscall.SIGFPE, + syscall.SIGILL, + syscall.SIGQUIT, + syscall.SIGSEGV, + syscall.SIGTERM, + } + sigChan := make(chan os.Signal, len(fatalSigs)) + signal.Notify(sigChan, fatalSigs...) + lg := NewLogger("exit", cnf) + go func() { + sig := <-sigChan + lg.Errorf("Terminating on signal: %v\n", sig) + lg.Close() + os.Exit(1) + }() +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/query.go b/htrace-htraced/go/src/org/apache/htrace/common/query.go new file mode 100644 index 0000000..8c9128f --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/query.go @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "encoding/json" +) + +// +// Represents queries that can be sent to htraced. +// +// Each query consists of set of predicates that will be 'AND'ed together to +// return a set of spans. Predicates contain an operation, a field, and a +// value. +// +// For example, a query might be "return the first 100 spans between 5:00pm +// and 5:01pm" This query would have two predicates: time greater than or +// equal to 5:00pm, and time less than or equal to 5:01pm. +// In HTrace, times are always expressed in milliseconds since the Epoch. +// So this would become: +// { "lim" : 100, "pred" : [ +// { "op" : "ge", "field" : "begin", "val" : 1234 }, +// { "op" : "le", "field" : "begin", "val" : 5678 }, +// ] } +// +// Where '1234' and '5678' were replaced by times since the epoch in +// milliseconds. +// + +type Op string + +const ( + CONTAINS Op = "cn" + EQUALS Op = "eq" + LESS_THAN_OR_EQUALS Op = "le" + GREATER_THAN_OR_EQUALS Op = "ge" + GREATER_THAN Op = "gt" +) + +func (op Op) IsDescending() bool { + return op == LESS_THAN_OR_EQUALS +} + +func (op Op) IsValid() bool { + ops := ValidOps() + for i := range ops { + if ops[i] == op { + return true + } + } + return false +} + +func ValidOps() []Op { + return []Op{CONTAINS, EQUALS, LESS_THAN_OR_EQUALS, GREATER_THAN_OR_EQUALS, + GREATER_THAN} +} + +type Field string + +const ( + SPAN_ID Field = "spanid" + DESCRIPTION Field = "description" + BEGIN_TIME Field = "begin" + END_TIME Field = "end" + DURATION Field = "duration" +) + +func (field Field) IsValid() bool { + fields := ValidFields() + for i := range fields { + if fields[i] == field { + return true + } + } + return false +} + +func ValidFields() []Field { + return []Field{SPAN_ID, DESCRIPTION, BEGIN_TIME, END_TIME, DURATION} +} + +type Predicate struct { + Op Op `json:"op"` + Field Field `json:"field"` + Val string `val:"val"` +} + +func (pred *Predicate) String() string { + buf, err := json.Marshal(pred) + if err != nil { + panic(err) + } + return string(buf) +} + +type Query struct { + Predicates []Predicate `json:"pred"` + Lim int `json:"lim"` + Prev *Span `json:"prev"` +} + +func (query *Query) String() string { + buf, err := json.Marshal(query) + if err != nil { + panic(err) + } + return string(buf) +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/query_test.go b/htrace-htraced/go/src/org/apache/htrace/common/query_test.go new file mode 100644 index 0000000..2697d9c --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/query_test.go @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "testing" +) + +func TestValidOps(t *testing.T) { + for i := range ValidOps() { + op := ValidOps()[i] + if !op.IsValid() { + t.Fatalf("op %s was in ValidOps, but IsValid returned false.\n", op) + } + } + invalidOp := Op("completelybogus") + if invalidOp.IsValid() { + t.Fatalf("op %s was invalid, but IsValid returned true.\n", invalidOp) + } +} + +func TestValidFields(t *testing.T) { + for i := range ValidFields() { + field := ValidFields()[i] + if !field.IsValid() { + t.Fatalf("field %s was in ValidFields, but IsValid returned false.\n", field) + } + } + invalidField := Field("completelybogus") + if invalidField.IsValid() { + t.Fatalf("field %s was invalid, but IsValid returned true.\n", invalidField) + } +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/rest.go b/htrace-htraced/go/src/org/apache/htrace/common/rest.go new file mode 100644 index 0000000..b898ca4 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/rest.go @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +// Info returned by /server/info +type ServerInfo struct { + // The server release version. + ReleaseVersion string + + // The git hash that this software was built with. + GitVersion string +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/rpc.go b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go new file mode 100644 index 0000000..fe50a44 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/rpc.go @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +// The 4-byte magic number which is sent first in the HRPC header +const HRPC_MAGIC = 0x43525448 + +// Method ID codes. Do not reorder these. +const ( + METHOD_ID_NONE = 0 + METHOD_ID_WRITE_SPANS = iota +) + +const METHOD_NAME_WRITE_SPANS = "HrpcHandler.WriteSpans" + +// Maximum length of the error message passed in an HRPC response +const MAX_HRPC_ERROR_LENGTH = 4 * 1024 * 1024 + +// Maximum length of HRPC message body +const MAX_HRPC_BODY_LENGTH = 64 * 1024 * 1024 + +// A request to write spans to htraced. +type WriteSpansReq struct { + DefaultPid string + Spans []*Span +} + +// A response to a WriteSpansReq +type WriteSpansResp struct { +} + +// The header which is sent over the wire for HRPC +type HrpcRequestHeader struct { + Magic uint32 + MethodId uint32 + Seq uint64 + Length uint32 +} + +// The response which is sent over the wire for HRPC +type HrpcResponseHeader struct { + Seq uint64 + MethodId uint32 + ErrLength uint32 + Length uint32 +} + +func HrpcMethodIdToMethodName(id uint32) string { + switch id { + case METHOD_ID_WRITE_SPANS: + return METHOD_NAME_WRITE_SPANS + default: + return "" + } +} + +func HrpcMethodNameToId(name string) uint32 { + switch name { + case METHOD_NAME_WRITE_SPANS: + return METHOD_ID_WRITE_SPANS + default: + return METHOD_ID_NONE + } +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/span.go b/htrace-htraced/go/src/org/apache/htrace/common/span.go new file mode 100644 index 0000000..b276844 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/span.go @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" +) + +// +// Represents a trace span. +// +// Compatibility notes: +// When converting to JSON, we store the 64-bit numbers as hexadecimal strings rather than as +// integers. This is because JavaScript lacks the ability to handle 64-bit integers. Numbers above +// about 55 bits will be rounded by Javascript. Since the Javascript UI is a primary consumer of +// this JSON data, we have to simply pass it as a string. +// + +type TraceInfoMap map[string]string + +type TimelineAnnotation struct { + Time int64 `json:"t"` + Msg string `json:"m"` +} + +type SpanId uint64 + +func (id SpanId) String() string { + return fmt.Sprintf("%016x", uint64(id)) +} + +func (id SpanId) Val() uint64 { + return uint64(id) +} + +func (id SpanId) MarshalJSON() ([]byte, error) { + return []byte(`"` + fmt.Sprintf("%016x", uint64(id)) + `"`), nil +} + +type SpanSlice []*Span + +func (s SpanSlice) Len() int { + return len(s) +} + +func (s SpanSlice) Less(i, j int) bool { + return s[i].Id < s[j].Id +} + +func (s SpanSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +type SpanIdSlice []SpanId + +func (s SpanIdSlice) Len() int { + return len(s) +} + +func (s SpanIdSlice) Less(i, j int) bool { + return s[i] < s[j] +} + +func (s SpanIdSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +const DOUBLE_QUOTE = 0x22 + +func (id *SpanId) UnmarshalJSON(b []byte) error { + if b[0] != DOUBLE_QUOTE { + return errors.New("Expected spanID to start with a string quote.") + } + if b[len(b)-1] != DOUBLE_QUOTE { + return errors.New("Expected spanID to end with a string quote.") + } + return id.FromString(string(b[1 : len(b)-1])) +} + +func (id *SpanId) FromString(str string) error { + v, err := strconv.ParseUint(str, 16, 64) + if err != nil { + return err + } + *id = SpanId(v) + return nil +} + +type SpanData struct { + Begin int64 `json:"b"` + End int64 `json:"e"` + Description string `json:"d"` + TraceId SpanId `json:"i"` + Parents []SpanId `json:"p"` + Info TraceInfoMap `json:"n,omitempty"` + ProcessId string `json:"r"` + TimelineAnnotations []TimelineAnnotation `json:"t,omitempty"` +} + +type Span struct { + Id SpanId `json:"s"` + SpanData +} + +func (span *Span) ToJson() []byte { + jbytes, err := json.Marshal(*span) + if err != nil { + panic(err) + } + return jbytes +} + +func (span *Span) String() string { + return string(span.ToJson()) +} + +// Compute the span duration. We ignore overflow since we never deal with negative times. +func (span *Span) Duration() int64 { + return span.End - span.Begin +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/span_test.go b/htrace-htraced/go/src/org/apache/htrace/common/span_test.go new file mode 100644 index 0000000..f218b3a --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/span_test.go @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "testing" +) + +func TestSpanToJson(t *testing.T) { + t.Parallel() + span := Span{Id: 2305843009213693952, + SpanData: SpanData{ + Begin: 123, + End: 456, + Description: "getFileDescriptors", + TraceId: 999, + Parents: []SpanId{}, + ProcessId: "testProcessId", + }} + ExpectStrEqual(t, + `{"s":"2000000000000000","b":123,"e":456,"d":"getFileDescriptors","i":"00000000000003e7","p":[],"r":"testProcessId"}`, + string(span.ToJson())) +} + +func TestAnnotatedSpanToJson(t *testing.T) { + t.Parallel() + span := Span{Id: 1305813009213693952, + SpanData: SpanData{ + Begin: 1234, + End: 4567, + Description: "getFileDescriptors2", + TraceId: 999, + Parents: []SpanId{}, + ProcessId: "testAnnotatedProcessId", + TimelineAnnotations: []TimelineAnnotation{ + TimelineAnnotation{ + Time: 7777, + Msg: "contactedServer", + }, + TimelineAnnotation{ + Time: 8888, + Msg: "passedFd", + }, + }, + }} + ExpectStrEqual(t, + `{"s":"121f2e036d442000","b":1234,"e":4567,"d":"getFileDescriptors2","i":"00000000000003e7","p":[],"r":"testAnnotatedProcessId","t":[{"t":7777,"m":"contactedServer"},{"t":8888,"m":"passedFd"}]}`, + string(span.ToJson())) +} diff --git a/htrace-htraced/go/src/org/apache/htrace/common/test_util.go b/htrace-htraced/go/src/org/apache/htrace/common/test_util.go new file mode 100644 index 0000000..871c847 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/common/test_util.go @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package common + +import ( + "fmt" + "testing" + "time" +) + +type Int64Slice []int64 + +func (p Int64Slice) Len() int { return len(p) } +func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +type SupplierFun func() bool + +// +// Wait for a configurable amount of time for a precondition to become true. +// +// Example: +// WaitFor(time.Minute * 1, time.Millisecond * 1, func() bool { +// return ht.Store.GetStatistics().NumSpansWritten >= 3 +// }) +// +func WaitFor(dur time.Duration, poll time.Duration, fun SupplierFun) { + if poll == 0 { + poll = dur / 10 + } + if poll <= 0 { + panic("Can't have a polling time less than zero.") + } + endTime := time.Now().Add(dur) + for { + if fun() { + return + } + if !time.Now().Before(endTime) { + break + } + time.Sleep(poll) + } + panic(fmt.Sprintf("Timed out after %s", dur)) +} + +// Trigger a test failure if two strings are not equal. +func ExpectStrEqual(t *testing.T, expect string, actual string) { + if expect != actual { + t.Fatalf("Expected:\n%s\nGot:\n%s\n", expect, actual) + } +} + +// Trigger a test failure if the JSON representation of two spans are not equals. +func ExpectSpansEqual(t *testing.T, spanA *Span, spanB *Span) { + ExpectStrEqual(t, string(spanA.ToJson()), string(spanB.ToJson())) +} diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/config.go b/htrace-htraced/go/src/org/apache/htrace/conf/config.go new file mode 100644 index 0000000..6093649 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/conf/config.go @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package conf + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" +) + +// +// The configuration code for HTraced. +// +// HTraced can be configured via Hadoop-style XML configuration files, or by passing -Dkey=value +// command line arguments. Command-line arguments without an equals sign, such as "-Dkey", will be +// treated as setting the key to "true". +// +// Configuration key constants should be defined in config_keys.go. Each key should have a default, +// which will be used if the user supplies no value, or supplies an invalid value. +// For that reason, it is not necessary for the Get, GetInt, etc. functions to take a default value +// argument. +// +// Configuration objects are immutable. However, you can make a copy of a configuration which adds +// some changes using Configuration#Clone(). +// + +type Config struct { + settings map[string]string + defaults map[string]string +} + +type Builder struct { + // If non-nil, the XML configuration file to read. + Reader io.Reader + + // If non-nil, the configuration values to use. + Values map[string]string + + // If non-nil, the default configuration values to use. + Defaults map[string]string + + // If non-nil, the command-line arguments to use. + Argv []string +} + +func getHTracedConfDirs(dlog io.Writer) []string { + confDir := os.Getenv("HTRACED_CONF_DIR") + io.WriteString(dlog, fmt.Sprintf("HTRACED_CONF_DIR=%s\n", confDir)) + paths := filepath.SplitList(confDir) + if len(paths) < 1 { + return []string{"."} + } + return paths +} + +// Load a configuration from the application's argv, configuration file, and the standard +// defaults. +func LoadApplicationConfig() (*Config, io.Reader) { + dlog := new(bytes.Buffer) + reader := openFile(CONFIG_FILE_NAME, getHTracedConfDirs(dlog), dlog) + bld := Builder{} + if reader != nil { + defer reader.Close() + bld.Reader = bufio.NewReader(reader) + } + bld.Argv = os.Args[1:] + bld.Defaults = DEFAULTS + cnf, err := bld.Build() + if err != nil { + log.Fatal("Error building configuration: " + err.Error()) + } + os.Args = append(os.Args[0:1], bld.Argv...) + keys := make(sort.StringSlice, 0, 20) + for k, _ := range cnf.settings { + keys = append(keys, k) + } + sort.Sort(keys) + for i := range keys { + io.WriteString(dlog, fmt.Sprintf("%s = %s\n", + keys[i], cnf.settings[keys[i]])) + } + return cnf, dlog +} + +// Attempt to open a configuration file somewhere on the provided list of paths. +func openFile(cnfName string, paths []string, dlog io.Writer) io.ReadCloser { + for p := range paths { + path := fmt.Sprintf("%s%c%s", paths[p], os.PathSeparator, cnfName) + file, err := os.Open(path) + if err == nil { + io.WriteString(dlog, fmt.Sprintf("Reading configuration from %s.\n", path)) + return file + } + if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOENT { + continue + } + io.WriteString(dlog, fmt.Sprintf("Error opening %s for read: %s\n", path, err.Error())) + } + return nil +} + +// Try to parse a command-line element as a key=value pair. +func parseAsConfigFlag(flag string) (string, string) { + var confPart string + if strings.HasPrefix(flag, "-D") { + confPart = flag[2:] + } else if strings.HasPrefix(flag, "--D") { + confPart = flag[3:] + } else { + return "", "" + } + if len(confPart) == 0 { + return "", "" + } + idx := strings.Index(confPart, "=") + if idx == -1 { + return confPart, "true" + } + return confPart[0:idx], confPart[idx+1:] +} + +// Build a new configuration object from the provided conf.Builder. +func (bld *Builder) Build() (*Config, error) { + // Load values and defaults + cnf := Config{} + cnf.settings = make(map[string]string) + if bld.Values != nil { + for k, v := range bld.Values { + cnf.settings[k] = v + } + } + cnf.defaults = make(map[string]string) + if bld.Defaults != nil { + for k, v := range bld.Defaults { + cnf.defaults[k] = v + } + } + + // Process the configuration file, if we have one + if bld.Reader != nil { + parseXml(bld.Reader, cnf.settings) + } + + // Process command line arguments + var i int + for i < len(bld.Argv) { + str := bld.Argv[i] + key, val := parseAsConfigFlag(str) + if key != "" { + if val == "" { + cnf.settings[key] = "true" + } else { + cnf.settings[key] = val + } + bld.Argv = append(bld.Argv[:i], bld.Argv[i+1:]...) + } else { + i++ + } + } + return &cnf, nil +} + +// Returns true if the configuration has a non-default value for the given key. +func (cnf *Config) Contains(key string) bool { + _, ok := cnf.settings[key] + return ok +} + +// Get a string configuration key. +func (cnf *Config) Get(key string) string { + ret := cnf.settings[key] + if ret != "" { + return ret + } + return cnf.defaults[key] +} + +// Get a boolean configuration key. +func (cnf *Config) GetBool(key string) bool { + str := cnf.settings[key] + ret, err := strconv.ParseBool(str) + if err == nil { + return ret + } + str = cnf.defaults[key] + ret, err = strconv.ParseBool(str) + if err == nil { + return ret + } + return false +} + +// Get an integer configuration key. +func (cnf *Config) GetInt(key string) int { + str := cnf.settings[key] + ret, err := strconv.Atoi(str) + if err == nil { + return ret + } + str = cnf.defaults[key] + ret, err = strconv.Atoi(str) + if err == nil { + return ret + } + return 0 +} + +// Get an int64 configuration key. +func (cnf *Config) GetInt64(key string) int64 { + str := cnf.settings[key] + ret, err := strconv.ParseInt(str, 10, 64) + if err == nil { + return ret + } + str = cnf.defaults[key] + ret, err = strconv.ParseInt(str, 10, 64) + if err == nil { + return ret + } + return 0 +} + +// Make a deep copy of the given configuration. +// Optionally, you can specify particular key/value pairs to change. +// Example: +// cnf2 := cnf.Copy("my.changed.key", "my.new.value") +func (cnf *Config) Clone(args ...string) *Config { + if len(args)%2 != 0 { + panic("The arguments to Config#copy are key1, value1, " + + "key2, value2, and so on. You must specify an even number of arguments.") + } + ncnf := &Config{defaults: cnf.defaults} + ncnf.settings = make(map[string]string) + for k, v := range cnf.settings { + ncnf.settings[k] = v + } + for i := 0; i < len(args); i += 2 { + ncnf.settings[args[i]] = args[i+1] + } + return ncnf +} diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go b/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go new file mode 100644 index 0000000..ccb09e0 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package conf + +import ( + "fmt" + "os" +) + +// +// Configuration keys for HTrace. +// + +// The platform-specific path separator. Usually slash. +var PATH_SEP string = fmt.Sprintf("%c", os.PathSeparator) + +// The platform-specific path list separator. Usually colon. +var PATH_LIST_SEP string = fmt.Sprintf("%c", os.PathListSeparator) + +// The name of the XML configuration file to look for. +const CONFIG_FILE_NAME = "htraced-conf.xml" + +// An environment variable containing a list of paths to search for the +// configuration file in. +const HTRACED_CONF_DIR = "HTRACED_CONF_DIR" + +// The web address to start the REST server on. +const HTRACE_WEB_ADDRESS = "web.address" + +// The default port for the Htrace web address. +const HTRACE_WEB_ADDRESS_DEFAULT_PORT = 9095 + +// The web address to start the REST server on. +const HTRACE_HRPC_ADDRESS = "hrpc.address" + +// The default port for the Htrace HRPC address. +const HTRACE_HRPC_ADDRESS_DEFAULT_PORT = 9075 + +// The directories to put the data store into. Separated by PATH_LIST_SEP. +const HTRACE_DATA_STORE_DIRECTORIES = "data.store.directories" + +// Boolean key which indicates whether we should clear data on startup. +const HTRACE_DATA_STORE_CLEAR = "data.store.clear" + +// How many writes to buffer before applying backpressure to span senders. +const HTRACE_DATA_STORE_SPAN_BUFFER_SIZE = "data.store.span.buffer.size" + +// Path to put the logs from htrace, or the empty string to use stdout. +const HTRACE_LOG_PATH = "log.path" + +// The log level to use for the logs in htrace. +const HTRACE_LOG_LEVEL = "log.level" + +// A host:port pair to send information to on startup. This is used in unit +// tests to determine the (random) port of the htraced process that has been +// started. +const HTRACE_STARTUP_NOTIFICATION_ADDRESS = "startup.notification.address" + +// Default values for HTrace configuration keys. +var DEFAULTS = map[string]string{ + HTRACE_WEB_ADDRESS: fmt.Sprintf("0.0.0.0:%d", HTRACE_WEB_ADDRESS_DEFAULT_PORT), + HTRACE_HRPC_ADDRESS: fmt.Sprintf("0.0.0.0:%d", HTRACE_HRPC_ADDRESS_DEFAULT_PORT), + HTRACE_DATA_STORE_DIRECTORIES: PATH_SEP + "tmp" + PATH_SEP + "htrace1" + + PATH_LIST_SEP + PATH_SEP + "tmp" + PATH_SEP + "htrace2", + HTRACE_DATA_STORE_CLEAR: "false", + HTRACE_DATA_STORE_SPAN_BUFFER_SIZE: "100", + HTRACE_LOG_PATH: "", + HTRACE_LOG_LEVEL: "INFO", +} diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/config_test.go b/htrace-htraced/go/src/org/apache/htrace/conf/config_test.go new file mode 100644 index 0000000..42c1c71 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/conf/config_test.go @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package conf + +import ( + "bytes" + "os" + "strings" + "testing" +) + +// Test that parsing command-line arguments of the form -Dfoo=bar works. +func TestParseArgV(t *testing.T) { + t.Parallel() + argv := []string{"-Dfoo=bar", "-Dbaz=123", "-DsillyMode"} + bld := &Builder{Argv: argv} + cnf, err := bld.Build() + if err != nil { + t.Fatal() + } + if "bar" != cnf.Get("foo") { + t.Fatal() + } + if 123 != cnf.GetInt("baz") { + t.Fatal() + } + if !cnf.GetBool("sillyMode") { + t.Fatal() + } + if cnf.GetBool("otherSillyMode") { + t.Fatal() + } +} + +// Test that default values work. +// Defaults are used only when the configuration option is not present or can't be parsed. +func TestDefaults(t *testing.T) { + t.Parallel() + argv := []string{"-Dfoo=bar", "-Dbaz=invalidNumber"} + defaults := map[string]string{ + "foo": "notbar", + "baz": "456", + "foo2": "4611686018427387904", + } + bld := &Builder{Argv: argv, Defaults: defaults} + cnf, err := bld.Build() + if err != nil { + t.Fatal() + } + if "bar" != cnf.Get("foo") { + t.Fatal() + } + if 456 != cnf.GetInt("baz") { + t.Fatal() + } + if 4611686018427387904 != cnf.GetInt64("foo2") { + t.Fatal() + } +} + +// Test that we can parse our XML configuration file. +func TestXmlConfigurationFile(t *testing.T) { + t.Parallel() + xml := ` + + + + + foo.bar + 123 + + + foo.baz + xmlValue + + + +` + xmlReader := strings.NewReader(xml) + argv := []string{"-Dfoo.bar=456"} + defaults := map[string]string{ + "foo.bar": "789", + "cmdline.opt": "4611686018427387904", + } + bld := &Builder{Argv: argv, Defaults: defaults, Reader: xmlReader} + cnf, err := bld.Build() + if err != nil { + t.Fatal() + } + // The command-line argument takes precedence over the XML and the defaults. + if 456 != cnf.GetInt("foo.bar") { + t.Fatal() + } + if "xmlValue" != cnf.Get("foo.baz") { + t.Fatalf("foo.baz = %s", cnf.Get("foo.baz")) + } + if "" != cnf.Get("commented.out") { + t.Fatal() + } + if 4611686018427387904 != cnf.GetInt64("cmdline.opt") { + t.Fatal() + } +} + +// Test our handling of the HTRACE_CONF_DIR environment variable. +func TestGetHTracedConfDirs(t *testing.T) { + os.Setenv("HTRACED_CONF_DIR", "") + dlog := new(bytes.Buffer) + dirs := getHTracedConfDirs(dlog) + if len(dirs) != 1 || dirs[0] != "." { + t.Fatal() + } + os.Setenv("HTRACED_CONF_DIR", "/foo/bar:/baz") + dirs = getHTracedConfDirs(dlog) + if len(dirs) != 2 || dirs[0] != "/foo/bar" || dirs[1] != "/baz" { + t.Fatal() + } +} diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/xml.go b/htrace-htraced/go/src/org/apache/htrace/conf/xml.go new file mode 100644 index 0000000..de14bc5 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/conf/xml.go @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package conf + +import ( + "encoding/xml" + "io" + "log" +) + +type configuration struct { + Properties []propertyXml `xml:"property"` +} + +type propertyXml struct { + Name string `xml:"name"` + Value string `xml:"value"` +} + +// Parse an XML configuration file. +func parseXml(reader io.Reader, m map[string]string) error { + dec := xml.NewDecoder(reader) + configurationXml := configuration{} + err := dec.Decode(&configurationXml) + if err != nil { + return err + } + props := configurationXml.Properties + for p := range props { + key := props[p].Name + value := props[p].Value + if key == "" { + log.Println("Warning: ignoring element with missing or empty .") + continue + } + if value == "" { + log.Println("Warning: ignoring element with key " + key + " with missing or empty .") + continue + } + //log.Printf("setting %s to %s\n", key, value) + m[key] = value + } + return nil +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go b/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go new file mode 100644 index 0000000..38cdb58 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htrace/cmd.go @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/alecthomas/kingpin" + "io" + htrace "org/apache/htrace/client" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "os" + "time" +) + +var RELEASE_VERSION string +var GIT_VERSION string + +const EXIT_SUCCESS = 0 +const EXIT_FAILURE = 1 + +var verbose *bool + +const USAGE = `The Apache HTrace command-line tool. This tool retrieves and modifies settings and +other data on a running htraced daemon. + +If we find an ` + conf.CONFIG_FILE_NAME + ` configuration file in the list of directories +specified in ` + conf.HTRACED_CONF_DIR + `, we will use that configuration; otherwise, +the defaults will be used. +` + +func main() { + // Load htraced configuration + cnf := common.LoadApplicationConfig() + + // Parse argv + app := kingpin.New(os.Args[0], USAGE) + app.Flag("Dmy.key", "Set configuration key 'my.key' to 'my.value'. Replace 'my.key' "+ + "with any key you want to set.").Default("my.value").String() + addr := app.Flag("addr", "Server address.").String() + verbose = app.Flag("verbose", "Verbose.").Default("false").Bool() + version := app.Command("version", "Print the version of this program.") + serverInfo := app.Command("serverInfo", "Print information retrieved from an htraced server.") + findSpan := app.Command("findSpan", "Print information about a trace span with a given ID.") + findSpanId := findSpan.Arg("id", "Span ID to find. Example: 0x123456789abcdef").Required().Uint64() + findChildren := app.Command("findChildren", "Print out the span IDs that are children of a given span ID.") + parentSpanId := findChildren.Arg("id", "Span ID to print children for. Example: 0x123456789abcdef"). + Required().Uint64() + childLim := findChildren.Flag("lim", "Maximum number of child IDs to print.").Default("20").Int() + loadFile := app.Command("loadFile", "Write whitespace-separated JSON spans from a file to the server.") + loadFilePath := loadFile.Arg("path", + "A file containing whitespace-separated span JSON.").Required().String() + loadJson := app.Command("load", "Write JSON spans from the command-line to the server.") + loadJsonArg := loadJson.Arg("json", "A JSON span to write to the server.").Required().String() + dumpAll := app.Command("dumpAll", "Dump all spans from the htraced daemon.") + dumpAllOutPath := dumpAll.Arg("path", "The path to dump the trace spans to.").Default("-").String() + dumpAllLim := dumpAll.Flag("lim", "The number of spans to transfer from the server at once."). + Default("100").Int() + graph := app.Command("graph", "Visualize span JSON as a graph.") + graphJsonFile := graph.Arg("input", "The JSON file to load").Required().String() + graphDotFile := graph.Flag("output", + "The path to write a GraphViz dotfile to. This file can be used as input to "+ + "GraphViz, in order to generate a pretty picture. See graphviz.org for more "+ + "information about generating pictures of graphs.").Default("-").String() + query := app.Command("query", "Send a query to htraced.") + queryLim := query.Flag("lim", "Maximum number of spans to retrieve.").Default("20").Int() + queryArg := query.Arg("query", "The query string to send. Query strings have the format "+ + "[TYPE] [OPERATOR] [CONST], joined by AND statements.").Required().String() + rawQuery := app.Command("rawQuery", "Send a raw JSON query to htraced.") + rawQueryArg := query.Arg("json", "The query JSON to send.").Required().String() + cmd := kingpin.MustParse(app.Parse(os.Args[1:])) + + // Add the command-line settings into the configuration. + if *addr != "" { + cnf = cnf.Clone(conf.HTRACE_WEB_ADDRESS, *addr) + } + + // Handle commands that don't require an HTrace client. + switch cmd { + case version.FullCommand(): + os.Exit(printVersion()) + case graph.FullCommand(): + err := jsonSpanFileToDotFile(*graphJsonFile, *graphDotFile) + if err != nil { + fmt.Printf("graphing error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + } + + // Create HTrace client + hcl, err := htrace.NewClient(cnf) + if err != nil { + fmt.Printf("Failed to create HTrace client: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + + // Handle commands that require an HTrace client. + switch cmd { + case version.FullCommand(): + os.Exit(printVersion()) + case serverInfo.FullCommand(): + os.Exit(printServerInfo(hcl)) + case findSpan.FullCommand(): + os.Exit(doFindSpan(hcl, common.SpanId(*findSpanId))) + case findChildren.FullCommand(): + os.Exit(doFindChildren(hcl, common.SpanId(*parentSpanId), *childLim)) + case loadJson.FullCommand(): + os.Exit(doLoadSpanJson(hcl, *loadJsonArg)) + case loadFile.FullCommand(): + os.Exit(doLoadSpanJsonFile(hcl, *loadFilePath)) + case dumpAll.FullCommand(): + err := doDumpAll(hcl, *dumpAllOutPath, *dumpAllLim) + if err != nil { + fmt.Printf("dumpAll error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + case query.FullCommand(): + err := doQueryFromString(hcl, *queryArg, *queryLim) + if err != nil { + fmt.Printf("query error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + case rawQuery.FullCommand(): + err := doRawQuery(hcl, *rawQueryArg) + if err != nil { + fmt.Printf("raw query error: %s\n", err.Error()) + os.Exit(EXIT_FAILURE) + } + os.Exit(EXIT_SUCCESS) + } + + app.UsageErrorf(os.Stderr, "You must supply a command to run.") +} + +// Print the version of the htrace binary. +func printVersion() int { + fmt.Printf("Running htrace command version %s.\n", RELEASE_VERSION) + return EXIT_SUCCESS +} + +// Print information retrieved from an htraced server via /server/info +func printServerInfo(hcl *htrace.Client) int { + info, err := hcl.GetServerInfo() + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + fmt.Printf("HTraced server version %s (%s)\n", info.ReleaseVersion, info.GitVersion) + return EXIT_SUCCESS +} + +// Print information about a trace span. +func doFindSpan(hcl *htrace.Client, sid common.SpanId) int { + span, err := hcl.FindSpan(sid) + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + if span == nil { + fmt.Printf("Span ID not found.\n") + return EXIT_FAILURE + } + pbuf, err := json.MarshalIndent(span, "", " ") + if err != nil { + fmt.Printf("Error: error pretty-printing span to JSON: %s\n", err.Error()) + return EXIT_FAILURE + } + fmt.Printf("%s\n", string(pbuf)) + return EXIT_SUCCESS +} + +func doLoadSpanJsonFile(hcl *htrace.Client, spanFile string) int { + if spanFile == "" { + fmt.Printf("You must specify the json file to load.\n") + return EXIT_FAILURE + } + file, err := OpenInputFile(spanFile) + if err != nil { + fmt.Printf("Failed to open %s: %s\n", spanFile, err.Error()) + return EXIT_FAILURE + } + defer file.Close() + return doLoadSpans(hcl, bufio.NewReader(file)) +} + +func doLoadSpanJson(hcl *htrace.Client, spanJson string) int { + return doLoadSpans(hcl, bytes.NewBufferString(spanJson)) +} + +func doLoadSpans(hcl *htrace.Client, reader io.Reader) int { + dec := json.NewDecoder(reader) + spans := make([]*common.Span, 0, 32) + var err error + for { + var span common.Span + if err = dec.Decode(&span); err != nil { + if err == io.EOF { + break + } + fmt.Printf("Failed to decode JSON: %s\n", err.Error()) + return EXIT_FAILURE + } + spans = append(spans, &span) + } + if *verbose { + fmt.Printf("Writing ") + prefix := "" + for i := range spans { + fmt.Printf("%s%s", prefix, spans[i].ToJson()) + prefix = ", " + } + fmt.Printf("\n") + } + err = hcl.WriteSpans(&common.WriteSpansReq{ + Spans: spans, + }) + if err != nil { + fmt.Println(err.Error()) + return EXIT_FAILURE + } + return EXIT_SUCCESS +} + +// Find information about the children of a span. +func doFindChildren(hcl *htrace.Client, sid common.SpanId, lim int) int { + spanIds, err := hcl.FindChildren(sid, lim) + if err != nil { + fmt.Printf("%s\n", err.Error()) + return EXIT_FAILURE + } + pbuf, err := json.MarshalIndent(spanIds, "", " ") + if err != nil { + fmt.Println("Error: error pretty-printing span IDs to JSON: %s", err.Error()) + return 1 + } + fmt.Printf("%s\n", string(pbuf)) + return 0 +} + +// Dump all spans from the htraced daemon. +func doDumpAll(hcl *htrace.Client, outPath string, lim int) error { + file, err := CreateOutputFile(outPath) + if err != nil { + return err + } + w := bufio.NewWriter(file) + defer func() { + if file != nil { + w.Flush() + file.Close() + } + }() + out := make(chan *common.Span, 50) + var dumpErr error + go func() { + dumpErr = hcl.DumpAll(lim, out) + }() + var numSpans int64 + nextLogTime := time.Now().Add(time.Second * 5) + for { + span, channelOpen := <-out + if !channelOpen { + break + } + if err == nil { + _, err = fmt.Fprintf(w, "%s\n", span.ToJson()) + } + if *verbose { + numSpans++ + now := time.Now() + if !now.Before(nextLogTime) { + nextLogTime = now.Add(time.Second * 5) + fmt.Printf("received %d span(s)...\n", numSpans) + } + } + } + if err != nil { + return errors.New(fmt.Sprintf("Write error %s", err.Error())) + } + if dumpErr != nil { + return errors.New(fmt.Sprintf("Dump error %s", dumpErr.Error())) + } + err = w.Flush() + if err != nil { + return err + } + err = file.Close() + file = nil + if err != nil { + return err + } + return nil +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/file.go b/htrace-htraced/go/src/org/apache/htrace/htrace/file.go new file mode 100644 index 0000000..ea214be --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htrace/file.go @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "org/apache/htrace/common" + "os" +) + +// A file used for input. +// Transparently supports using stdin for input. +type InputFile struct { + *os.File + path string +} + +// Open an input file. Stdin will be used when path is - +func OpenInputFile(path string) (*InputFile, error) { + if path == "-" { + return &InputFile{File: os.Stdin, path: path}, nil + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + return &InputFile{File: file, path: path}, nil +} + +func (file *InputFile) Close() { + if file.path != "-" { + file.File.Close() + } +} + +// A file used for output. +// Transparently supports using stdout for output. +type OutputFile struct { + *os.File + path string +} + +// Create an output file. Stdout will be used when path is - +func CreateOutputFile(path string) (*OutputFile, error) { + if path == "-" { + return &OutputFile{File: os.Stdout, path: path}, nil + } + file, err := os.Create(path) + if err != nil { + return nil, err + } + return &OutputFile{File: file, path: path}, nil +} + +func (file *OutputFile) Close() error { + if file.path != "-" { + return file.File.Close() + } + return nil +} + +// FailureDeferringWriter is a writer which allows us to call Printf multiple +// times and then check if all the printfs succeeded at the very end, rather +// than checking after each call. We will not attempt to write more data +// after the first write failure. +type FailureDeferringWriter struct { + io.Writer + err error +} + +func NewFailureDeferringWriter(writer io.Writer) *FailureDeferringWriter { + return &FailureDeferringWriter{writer, nil} +} + +func (w *FailureDeferringWriter) Printf(format string, v ...interface{}) { + if w.err != nil { + return + } + str := fmt.Sprintf(format, v...) + _, err := w.Writer.Write([]byte(str)) + if err != nil { + w.err = err + } +} + +func (w *FailureDeferringWriter) Error() error { + return w.err +} + +// Read a file full of whitespace-separated span JSON into a slice of spans. +func readSpansFile(path string) (common.SpanSlice, error) { + file, err := OpenInputFile(path) + if err != nil { + return nil, err + } + defer file.Close() + return readSpans(bufio.NewReader(file)) +} + +// Read whitespace-separated span JSON into a slice of spans. +func readSpans(reader io.Reader) (common.SpanSlice, error) { + spans := make(common.SpanSlice, 0) + dec := json.NewDecoder(reader) + for { + var span common.Span + err := dec.Decode(&span) + if err != nil { + if err != io.EOF { + return nil, errors.New(fmt.Sprintf("Decode error after decoding %d "+ + "span(s): %s", len(spans), err.Error())) + } + break + } + spans = append(spans, &span) + } + return spans, nil +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/file_test.go b/htrace-htraced/go/src/org/apache/htrace/htrace/file_test.go new file mode 100644 index 0000000..b6f9cac --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htrace/file_test.go @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "errors" + "io" + "io/ioutil" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "org/apache/htrace/test" + "os" + "strings" + "testing" +) + +func TestInputFileAndOutputFile(t *testing.T) { + tdir, err := ioutil.TempDir(os.TempDir(), "TestInputFileAndOutputFile") + if err != nil { + t.Fatalf("failed to create TempDir: %s\n", err.Error()) + } + defer os.RemoveAll(tdir) + tpath := tdir + conf.PATH_SEP + "test" + var ofile *OutputFile + ofile, err = CreateOutputFile(tpath) + if err != nil { + t.Fatalf("failed to create OutputFile at %s: %s\n", tpath, err.Error()) + } + defer func() { + if ofile != nil { + ofile.Close() + } + }() + w := NewFailureDeferringWriter(ofile) + w.Printf("Hello, world!\n") + w.Printf("2 + 2 = %d\n", 4) + if w.Error() != nil { + t.Fatalf("got unexpected error writing to %s: %s\n", tpath, w.Error().Error()) + } + err = ofile.Close() + ofile = nil + if err != nil { + t.Fatalf("error on closing OutputFile for %s: %s\n", tpath, err.Error()) + } + var ifile *InputFile + ifile, err = OpenInputFile(tpath) + defer ifile.Close() + expected := "Hello, world!\n2 + 2 = 4\n" + buf := make([]byte, len(expected)) + _, err = io.ReadAtLeast(ifile, buf, len(buf)) + if err != nil { + t.Fatalf("unexpected error on reading %s: %s\n", tpath, err.Error()) + } + str := string(buf) + if str != expected { + t.Fatalf("Could not read back what we wrote to %s.\n"+ + "Got:\n%s\nExpected:\n%s\n", tpath, str, expected) + } +} + +type LimitedBufferWriter struct { + buf []byte + off int +} + +const LIMITED_BUFFER_MESSAGE = "There isn't enough buffer to go around!" + +func (w *LimitedBufferWriter) Write(p []byte) (int, error) { + var nwritten int + for i := range p { + if w.off >= len(w.buf) { + return nwritten, errors.New(LIMITED_BUFFER_MESSAGE) + } + w.buf[w.off] = p[i] + w.off = w.off + 1 + nwritten++ + } + return nwritten, nil +} + +func TestFailureDeferringWriter(t *testing.T) { + lw := LimitedBufferWriter{buf: make([]byte, 20), off: 0} + w := NewFailureDeferringWriter(&lw) + w.Printf("Zippity do dah #%d\n", 1) + w.Printf("Zippity do dah #%d\n", 2) + if w.Error() == nil { + t.Fatalf("expected FailureDeferringWriter to experience a failure due to " + + "limited buffer size, but it did not.") + } + if w.Error().Error() != LIMITED_BUFFER_MESSAGE { + t.Fatalf("expected FailureDeferringWriter to have the error message %s, but "+ + "the message was %s\n", LIMITED_BUFFER_MESSAGE, w.Error().Error()) + } + expected := "Zippity do dah #1\nZi" + if string(lw.buf) != expected { + t.Fatalf("expected LimitedBufferWriter to contain %s, but it contained %s "+ + "instead.\n", expected, string(lw.buf)) + } +} + +func TestReadSpans(t *testing.T) { + SPAN_TEST_STR := `{"i":"bdd6d4ee48de59bf","s":"c0681027d3ea4928",` + + `"b":1424736225037,"e":1424736225901,"d":"ClientNamenodeProtocol#getFileInfo",` + + `"r":"FsShell","p":["60538dfb4df91418"]} +{"i":"bdd6d4ee48de59bf","s":"60538dfb4df91418","b":1424736224969,` + + `"e":1424736225960,"d":"getFileInfo","r":"FsShell","p":[],"n":{"path":"/"}} +` + r := strings.NewReader(SPAN_TEST_STR) + spans, err := readSpans(r) + if err != nil { + t.Fatalf("Failed to read spans from string via readSpans: %s\n", err.Error()) + } + SPAN_TEST_EXPECTED := common.SpanSlice{ + &common.Span{ + Id: test.SpanId("c0681027d3ea4928"), + SpanData: common.SpanData{ + TraceId: test.SpanId("bdd6d4ee48de59bf"), + Begin: 1424736225037, + End: 1424736225901, + Description: "ClientNamenodeProtocol#getFileInfo", + ProcessId: "FsShell", + Parents: []common.SpanId{test.SpanId("60538dfb4df91418")}, + }, + }, + &common.Span{ + Id: test.SpanId("60538dfb4df91418"), + SpanData: common.SpanData{ + TraceId: test.SpanId("bdd6d4ee48de59bf"), + Begin: 1424736224969, + End: 1424736225960, + Description: "getFileInfo", + ProcessId: "FsShell", + Parents: []common.SpanId{}, + Info: common.TraceInfoMap{ + "path": "/", + }, + }, + }, + } + if len(spans) != len(SPAN_TEST_EXPECTED) { + t.Fatalf("Expected %d spans, but got %d\n", + len(SPAN_TEST_EXPECTED), len(spans)) + } + for i := range SPAN_TEST_EXPECTED { + common.ExpectSpansEqual(t, spans[i], SPAN_TEST_EXPECTED[i]) + } +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/graph.go b/htrace-htraced/go/src/org/apache/htrace/htrace/graph.go new file mode 100644 index 0000000..dabf2df --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htrace/graph.go @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bufio" + "errors" + "fmt" + "io" + "org/apache/htrace/common" + "os" + "sort" +) + +// Create a dotfile from a json file. +func jsonSpanFileToDotFile(jsonFile string, dotFile string) error { + spans, err := readSpansFile(jsonFile) + if err != nil { + return errors.New(fmt.Sprintf("error reading %s: %s", + jsonFile, err.Error())) + } + var file *OutputFile + file, err = CreateOutputFile(dotFile) + if err != nil { + return errors.New(fmt.Sprintf("error opening %s for write: %s", + dotFile, err.Error())) + } + defer func() { + if file != nil { + file.Close() + } + }() + writer := bufio.NewWriter(file) + err = spansToDot(spans, writer) + if err != nil { + return err + } + err = writer.Flush() + if err != nil { + return err + } + err = file.Close() + file = nil + return err +} + +// Create output in dotfile format from a set of spans. +func spansToDot(spans common.SpanSlice, writer io.Writer) error { + sort.Sort(spans) + idMap := make(map[common.SpanId]*common.Span) + for i := range spans { + span := spans[i] + if idMap[span.Id] != nil { + fmt.Fprintf(os.Stderr, "There were multiple spans listed which "+ + "had ID %s.\nFirst:%s\nOther:%s\n", span.Id.String(), + idMap[span.Id].ToJson(), span.ToJson()) + } else { + idMap[span.Id] = span + } + } + childMap := make(map[common.SpanId]common.SpanSlice) + for i := range spans { + child := spans[i] + for j := range child.Parents { + parent := idMap[child.Parents[j]] + if parent == nil { + fmt.Fprintf(os.Stderr, "Can't find parent id %s for %s\n", + child.Parents[j].String(), child.ToJson()) + } else { + children := childMap[parent.Id] + if children == nil { + children = make(common.SpanSlice, 0) + } + children = append(children, child) + childMap[parent.Id] = children + } + } + } + w := NewFailureDeferringWriter(writer) + w.Printf("digraph spans {\n") + // Write out the nodes with their descriptions. + for i := range spans { + w.Printf(fmt.Sprintf(` "%s" [label="%s"];`+"\n", + spans[i].Id.String(), spans[i].Description)) + } + // Write out the edges between nodes... the parent/children relationships + for i := range spans { + children := childMap[spans[i].Id] + sort.Sort(children) + if children != nil { + for c := range children { + w.Printf(fmt.Sprintf(` "%s" -> "%s";`+"\n", + spans[i].Id.String(), children[c].Id)) + } + } + } + w.Printf("}\n") + return w.Error() +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/graph_test.go b/htrace-htraced/go/src/org/apache/htrace/htrace/graph_test.go new file mode 100644 index 0000000..8698a98 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htrace/graph_test.go @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bytes" + "org/apache/htrace/common" + "org/apache/htrace/test" + "testing" +) + +func TestSpansToDot(t *testing.T) { + TEST_SPANS := common.SpanSlice{ + &common.Span{ + Id: test.SpanId("6af3cc058e5d829d"), + SpanData: common.SpanData{ + TraceId: test.SpanId("0e4716fe911244de"), + Begin: 1424813349020, + End: 1424813349134, + Description: "newDFSInputStream", + ProcessId: "FsShell", + Parents: []common.SpanId{}, + Info: common.TraceInfoMap{ + "path": "/", + }, + }, + }, + &common.Span{ + Id: test.SpanId("75d16cc5b2c07d8a"), + SpanData: common.SpanData{ + TraceId: test.SpanId("0e4716fe911244de"), + Begin: 1424813349025, + End: 1424813349133, + Description: "getBlockLocations", + ProcessId: "FsShell", + Parents: []common.SpanId{test.SpanId("6af3cc058e5d829d")}, + }, + }, + &common.Span{ + Id: test.SpanId("e2c7273efb280a8c"), + SpanData: common.SpanData{ + TraceId: test.SpanId("0e4716fe911244de"), + Begin: 1424813349027, + End: 1424813349073, + Description: "ClientNamenodeProtocol#getBlockLocations", + ProcessId: "FsShell", + Parents: []common.SpanId{test.SpanId("75d16cc5b2c07d8a")}, + }, + }, + } + w := bytes.NewBuffer(make([]byte, 0, 2048)) + err := spansToDot(TEST_SPANS, w) + if err != nil { + t.Fatalf("spansToDot failed: error %s\n", err.Error()) + } + EXPECTED_STR := `digraph spans { + "6af3cc058e5d829d" [label="newDFSInputStream"]; + "75d16cc5b2c07d8a" [label="getBlockLocations"]; + "e2c7273efb280a8c" [label="ClientNamenodeProtocol#getBlockLocations"]; + "6af3cc058e5d829d" -> "75d16cc5b2c07d8a"; + "75d16cc5b2c07d8a" -> "e2c7273efb280a8c"; +} +` + if w.String() != EXPECTED_STR { + t.Fatalf("Expected to get:\n%s\nGot:\n%s\n", EXPECTED_STR, w.String()) + } +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htrace/queries.go b/htrace-htraced/go/src/org/apache/htrace/htrace/queries.go new file mode 100644 index 0000000..4ff246c --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htrace/queries.go @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "encoding/json" + "errors" + "fmt" + htrace "org/apache/htrace/client" + "org/apache/htrace/common" + "strings" + "unicode" +) + +// Convert a string into a whitespace-separated sequence of strings. +func tokenize(str string) []string { + prevQuote := rune(0) + f := func(c rune) bool { + switch { + case c == prevQuote: + prevQuote = rune(0) + return false + case prevQuote != rune(0): + return false + case unicode.In(c, unicode.Quotation_Mark): + prevQuote = c + return false + default: + return unicode.IsSpace(c) + } + } + return strings.FieldsFunc(str, f) +} + +// Parses a query string in the format of a series of +// [TYPE] [OPERATOR] [CONST] tuples, joined by AND statements. +type predicateParser struct { + tokens []string + curToken int +} + +func (ps *predicateParser) Parse() (*common.Predicate, error) { + if ps.curToken > len(ps.tokens) { + return nil, nil + } + if ps.curToken > 0 { + if strings.ToLower(ps.tokens[ps.curToken]) != "and" { + return nil, errors.New(fmt.Sprintf("Error parsing on token %d: "+ + "expected predicates to be joined by 'and', but found '%s'", + ps.curToken, ps.tokens[ps.curToken])) + } + ps.curToken++ + if ps.curToken > len(ps.tokens) { + return nil, errors.New(fmt.Sprintf("Nothing found after 'and' at "+ + "token %d", ps.curToken)) + } + } + field := common.Field(ps.tokens[ps.curToken]) + if !field.IsValid() { + return nil, errors.New(fmt.Sprintf("Invalid field specifier at token %d. "+ + "Can't understand %s. Valid field specifiers are %v", ps.curToken, + ps.tokens[ps.curToken], common.ValidFields())) + } + ps.curToken++ + if ps.curToken > len(ps.tokens) { + return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ + "at token %d", ps.curToken)) + } + op := common.Op(ps.tokens[ps.curToken]) + if !op.IsValid() { + return nil, errors.New(fmt.Sprintf("Invalid operation specifier at token %d. "+ + "Can't understand %s. Valid operation specifiers are %v", ps.curToken, + ps.tokens[ps.curToken], common.ValidOps())) + } + ps.curToken++ + if ps.curToken > len(ps.tokens) { + return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ + "at token %d", ps.curToken)) + } + val := ps.tokens[ps.curToken] + return &common.Predicate{Op: op, Field: field, Val: val}, nil +} + +func parseQueryString(str string) ([]common.Predicate, error) { + ps := predicateParser{tokens: tokenize(str)} + preds := make([]common.Predicate, 0) + for { + pred, err := ps.Parse() + if pred == nil { + break + } + if err != nil { + return nil, err + } + } + if len(preds) == 0 { + return nil, errors.New("Empty query string") + } + return preds, nil +} + +// Send a query from a query string. +func doQueryFromString(hcl *htrace.Client, str string, lim int) error { + query := &common.Query{Lim: lim} + var err error + query.Predicates, err = parseQueryString(str) + if err != nil { + return err + } + return doQuery(hcl, query) +} + +// Send a query from a raw JSON string. +func doRawQuery(hcl *htrace.Client, str string) error { + jsonBytes := []byte(str) + var query common.Query + err := json.Unmarshal(jsonBytes, &query) + if err != nil { + return errors.New(fmt.Sprintf("Error parsing provided JSON: %s\n", err.Error())) + } + return doQuery(hcl, &query) +} + +// Send a query. +func doQuery(hcl *htrace.Client, query *common.Query) error { + if *verbose { + qbytes, err := json.Marshal(*query) + if err != nil { + qbytes = []byte("marshaling error: " + err.Error()) + } + fmt.Printf("Sending query: %s\n", string(qbytes)) + } + spans, err := hcl.Query(query) + if err != nil { + return err + } + if *verbose { + fmt.Printf("%d results...\n", len(spans)) + } + for i := range spans { + fmt.Printf("%s\n", spans[i].ToJson()) + } + return nil +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/client_test.go b/htrace-htraced/go/src/org/apache/htrace/htraced/client_test.go new file mode 100644 index 0000000..218c1c8 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/client_test.go @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "fmt" + "math/rand" + htrace "org/apache/htrace/client" + "org/apache/htrace/common" + "org/apache/htrace/test" + "sort" + "testing" + "time" +) + +func TestClientGetServerInfo(t *testing.T) { + htraceBld := &MiniHTracedBuilder{Name: "TestClientGetServerInfo", + DataDirs: make([]string, 2)} + ht, err := htraceBld.Build() + if err != nil { + t.Fatalf("failed to create datastore: %s", err.Error()) + } + defer ht.Close() + var hcl *htrace.Client + hcl, err = htrace.NewClient(ht.ClientConf()) + if err != nil { + t.Fatalf("failed to create client: %s", err.Error()) + } + _, err = hcl.GetServerInfo() + if err != nil { + t.Fatalf("failed to call GetServerInfo: %s", err.Error()) + } +} + +func createRandomTestSpans(amount int) common.SpanSlice { + rnd := rand.New(rand.NewSource(2)) + allSpans := make(common.SpanSlice, amount) + allSpans[0] = test.NewRandomSpan(rnd, allSpans[0:0]) + for i := 1; i < amount; i++ { + allSpans[i] = test.NewRandomSpan(rnd, allSpans[1:i]) + } + allSpans[1].SpanData.Parents = []common.SpanId{common.SpanId(allSpans[0].Id)} + return allSpans +} + +func TestClientOperations(t *testing.T) { + htraceBld := &MiniHTracedBuilder{Name: "TestClientOperations", + DataDirs: make([]string, 2)} + ht, err := htraceBld.Build() + if err != nil { + t.Fatalf("failed to create datastore: %s", err.Error()) + } + defer ht.Close() + var hcl *htrace.Client + hcl, err = htrace.NewClient(ht.ClientConf()) + if err != nil { + t.Fatalf("failed to create client: %s", err.Error()) + } + + // Create some random trace spans. + NUM_TEST_SPANS := 30 + allSpans := createRandomTestSpans(NUM_TEST_SPANS) + + // Write half of the spans to htraced via the client. + err = hcl.WriteSpans(&common.WriteSpansReq{ + Spans: allSpans[0 : NUM_TEST_SPANS/2], + }) + if err != nil { + t.Fatalf("WriteSpans(0:%d) failed: %s\n", NUM_TEST_SPANS/2, + err.Error()) + } + + // Look up the first half of the spans. They should be found. + var span *common.Span + for i := 0; i < NUM_TEST_SPANS/2; i++ { + span, err = hcl.FindSpan(allSpans[i].Id) + if err != nil { + t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) + } + common.ExpectSpansEqual(t, allSpans[i], span) + } + + // Look up the second half of the spans. They should not be found. + for i := NUM_TEST_SPANS / 2; i < NUM_TEST_SPANS; i++ { + span, err = hcl.FindSpan(allSpans[i].Id) + if err != nil { + t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) + } + if span != nil { + t.Fatalf("Unexpectedly found a span we never write to "+ + "the server: FindSpan(%d) succeeded\n", i) + } + } + + // Test FindChildren + childSpan := allSpans[1] + parentId := childSpan.Parents[0] + var children []common.SpanId + children, err = hcl.FindChildren(parentId, 1) + if err != nil { + t.Fatalf("FindChildren(%s) failed: %s\n", parentId, err.Error()) + } + if len(children) != 1 { + t.Fatalf("FindChildren(%s) returned an invalid number of "+ + "children: expected %d, got %d\n", parentId, 1, len(children)) + } + if children[0] != childSpan.Id { + t.Fatalf("FindChildren(%s) returned an invalid child id: expected %s, "+ + " got %s\n", parentId, childSpan.Id, children[0]) + } + + // Test FindChildren on a span that has no children + childlessSpan := allSpans[NUM_TEST_SPANS/2] + children, err = hcl.FindChildren(childlessSpan.Id, 10) + if err != nil { + t.Fatalf("FindChildren(%d) failed: %s\n", childlessSpan.Id, err.Error()) + } + if len(children) != 0 { + t.Fatalf("FindChildren(%d) returned an invalid number of "+ + "children: expected %d, got %d\n", childlessSpan.Id, 0, len(children)) + } + + // Test Query + var query common.Query + query = common.Query{Lim: 10} + spans, err := hcl.Query(&query) + if err != nil { + t.Fatalf("Query({lim: %d}) failed: %s\n", 10, err.Error()) + } + if len(spans) != 10 { + t.Fatalf("Query({lim: %d}) returned an invalid number of "+ + "children: expected %d, got %d\n", 10, 10, len(spans)) + } +} + +func TestDumpAll(t *testing.T) { + htraceBld := &MiniHTracedBuilder{Name: "TestDumpAll", + DataDirs: make([]string, 2)} + ht, err := htraceBld.Build() + if err != nil { + t.Fatalf("failed to create datastore: %s", err.Error()) + } + defer ht.Close() + var hcl *htrace.Client + hcl, err = htrace.NewClient(ht.ClientConf()) + if err != nil { + t.Fatalf("failed to create client: %s", err.Error()) + } + + NUM_TEST_SPANS := 100 + allSpans := createRandomTestSpans(NUM_TEST_SPANS) + sort.Sort(allSpans) + err = hcl.WriteSpans(&common.WriteSpansReq{ + Spans: allSpans, + }) + if err != nil { + t.Fatalf("WriteSpans failed: %s\n", err.Error()) + } + out := make(chan *common.Span, 50) + var dumpErr error + go func() { + dumpErr = hcl.DumpAll(3, out) + }() + var numSpans int + nextLogTime := time.Now().Add(time.Millisecond * 5) + for { + span, channelOpen := <-out + if !channelOpen { + break + } + common.ExpectSpansEqual(t, allSpans[numSpans], span) + numSpans++ + if testing.Verbose() { + now := time.Now() + if !now.Before(nextLogTime) { + nextLogTime = now + nextLogTime = nextLogTime.Add(time.Millisecond * 5) + fmt.Printf("read back %d span(s)...\n", numSpans) + } + } + } + if numSpans != len(allSpans) { + t.Fatalf("expected to read %d spans... but only read %d\n", + len(allSpans), numSpans) + } + if dumpErr != nil { + t.Fatalf("got dump error %s\n", dumpErr.Error()) + } +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go new file mode 100644 index 0000000..0742555 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go @@ -0,0 +1,999 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bytes" + "encoding/gob" + "errors" + "fmt" + "github.com/jmhodges/levigo" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "os" + "strconv" + "strings" + "sync/atomic" +) + +// +// The data store code for HTraced. +// +// This code stores the trace spans. We use levelDB here so that we don't have to store everything +// in memory at all times. The data is sharded across multiple levelDB databases in multiple +// directories. Normally, these multiple directories will be on multiple disk drives. +// +// The main emphasis in the HTraceD data store is on quickly and efficiently storing trace span data +// coming from many daemons. Durability is not as big a concern as in some data stores, since +// losing a little bit of trace data if htraced goes down is not critical. We use the "gob" package +// for serialization. We assume that there will be many more writes than reads. +// +// Schema +// m -> dataStoreVersion +// s[8-byte-big-endian-sid] -> SpanData +// b[8-byte-big-endian-begin-time][8-byte-big-endian-child-sid] -> {} +// e[8-byte-big-endian-end-time][8-byte-big-endian-child-sid] -> {} +// d[8-byte-big-endian-duration][8-byte-big-endian-child-sid] -> {} +// p[8-byte-big-endian-parent-sid][8-byte-big-endian-child-sid] -> {} +// +// Note that span IDs are unsigned 64-bit numbers. +// Begin times, end times, and durations are signed 64-bit numbers. +// In order to get LevelDB to properly compare the signed 64-bit quantities, +// we flip the highest bit. This way, we can get leveldb to view negative +// quantities as less than non-negative ones. This also means that we can do +// all queries using unsigned 64-bit math, rather than having to special-case +// the signed fields. +// + +const UNKNOWN_LAYOUT_VERSION = 0 +const CURRENT_LAYOUT_VERSION = 2 + +var EMPTY_BYTE_BUF []byte = []byte{} + +const VERSION_KEY = 'v' +const SPAN_ID_INDEX_PREFIX = 's' +const BEGIN_TIME_INDEX_PREFIX = 'b' +const END_TIME_INDEX_PREFIX = 'e' +const DURATION_INDEX_PREFIX = 'd' +const PARENT_ID_INDEX_PREFIX = 'p' +const INVALID_INDEX_PREFIX = 0 + +type Statistics struct { + NumSpansWritten uint64 +} + +func (stats *Statistics) IncrementWrittenSpans() { + atomic.AddUint64(&stats.NumSpansWritten, 1) +} + +// Make a copy of the statistics structure, using atomic operations. +func (stats *Statistics) Copy() *Statistics { + return &Statistics{ + NumSpansWritten: atomic.LoadUint64(&stats.NumSpansWritten), + } +} + +// Translate an 8-byte value into a leveldb key. +func makeKey(tag byte, val uint64) []byte { + return []byte{ + tag, + byte(0xff & (val >> 56)), + byte(0xff & (val >> 48)), + byte(0xff & (val >> 40)), + byte(0xff & (val >> 32)), + byte(0xff & (val >> 24)), + byte(0xff & (val >> 16)), + byte(0xff & (val >> 8)), + byte(0xff & (val >> 0)), + } +} + +func keyToInt(key []byte) uint64 { + var id uint64 + id = (uint64(key[0]) << 56) | + (uint64(key[1]) << 48) | + (uint64(key[2]) << 40) | + (uint64(key[3]) << 32) | + (uint64(key[4]) << 24) | + (uint64(key[5]) << 16) | + (uint64(key[6]) << 8) | + (uint64(key[7]) << 0) + return id +} + +func makeSecondaryKey(tag byte, fir uint64, sec uint64) []byte { + return []byte{ + tag, + byte(0xff & (fir >> 56)), + byte(0xff & (fir >> 48)), + byte(0xff & (fir >> 40)), + byte(0xff & (fir >> 32)), + byte(0xff & (fir >> 24)), + byte(0xff & (fir >> 16)), + byte(0xff & (fir >> 8)), + byte(0xff & (fir >> 0)), + byte(0xff & (sec >> 56)), + byte(0xff & (sec >> 48)), + byte(0xff & (sec >> 40)), + byte(0xff & (sec >> 32)), + byte(0xff & (sec >> 24)), + byte(0xff & (sec >> 16)), + byte(0xff & (sec >> 8)), + byte(0xff & (sec >> 0)), + } +} + +// A single directory containing a levelDB instance. +type shard struct { + // The data store that this shard is part of + store *dataStore + + // The LevelDB instance. + ldb *levigo.DB + + // The path to the leveldb directory this shard is managing. + path string + + // Incoming requests to write Spans. + incoming chan *common.Span + + // The channel we will send a bool to when we exit. + exited chan bool +} + +// Process incoming spans for a shard. +func (shd *shard) processIncoming() { + lg := shd.store.lg + for { + span := <-shd.incoming + if span == nil { + lg.Infof("Shard processor for %s exiting.\n", shd.path) + shd.exited <- true + return + } + err := shd.writeSpan(span) + if err != nil { + lg.Errorf("Shard processor for %s got fatal error %s.\n", shd.path, err.Error()) + } else { + lg.Tracef("Shard processor for %s wrote span %s.\n", shd.path, span.ToJson()) + } + } +} + +// Convert a signed 64-bit number into an unsigned 64-bit number. We flip the +// highest bit, so that negative input values map to unsigned numbers which are +// less than non-negative input values. +func s2u64(val int64) uint64 { + ret := uint64(val) + ret ^= 0x8000000000000000 + return ret +} + +func (shd *shard) writeSpan(span *common.Span) error { + batch := levigo.NewWriteBatch() + defer batch.Close() + + // Add SpanData to batch. + spanDataBuf := new(bytes.Buffer) + spanDataEnc := gob.NewEncoder(spanDataBuf) + err := spanDataEnc.Encode(span.SpanData) + if err != nil { + return err + } + batch.Put(makeKey(SPAN_ID_INDEX_PREFIX, span.Id.Val()), spanDataBuf.Bytes()) + + // Add this to the parent index. + for parentIdx := range span.Parents { + batch.Put(makeSecondaryKey(PARENT_ID_INDEX_PREFIX, + span.Parents[parentIdx].Val(), span.Id.Val()), EMPTY_BYTE_BUF) + } + + // Add to the other secondary indices. + batch.Put(makeSecondaryKey(BEGIN_TIME_INDEX_PREFIX, s2u64(span.Begin), + span.Id.Val()), EMPTY_BYTE_BUF) + batch.Put(makeSecondaryKey(END_TIME_INDEX_PREFIX, s2u64(span.End), + span.Id.Val()), EMPTY_BYTE_BUF) + batch.Put(makeSecondaryKey(DURATION_INDEX_PREFIX, s2u64(span.Duration()), + span.Id.Val()), EMPTY_BYTE_BUF) + + err = shd.ldb.Write(shd.store.writeOpts, batch) + if err != nil { + return err + } + shd.store.stats.IncrementWrittenSpans() + if shd.store.WrittenSpans != nil { + shd.store.WrittenSpans <- span + } + return nil +} + +func (shd *shard) FindChildren(sid common.SpanId, childIds []common.SpanId, + lim int32) ([]common.SpanId, int32, error) { + searchKey := makeKey('p', sid.Val()) + iter := shd.ldb.NewIterator(shd.store.readOpts) + defer iter.Close() + iter.Seek(searchKey) + for { + if !iter.Valid() { + break + } + if lim == 0 { + break + } + key := iter.Key() + if !bytes.HasPrefix(key, searchKey) { + break + } + id := common.SpanId(keyToInt(key[9:])) + childIds = append(childIds, id) + lim-- + iter.Next() + } + return childIds, lim, nil +} + +// Close a shard. +func (shd *shard) Close() { + lg := shd.store.lg + shd.incoming <- nil + lg.Infof("Waiting for %s to exit...\n", shd.path) + if shd.exited != nil { + <-shd.exited + } + shd.ldb.Close() + lg.Infof("Closed %s...\n", shd.path) +} + +// The Data Store. +type dataStore struct { + lg *common.Logger + + // The shards which manage our LevelDB instances. + shards []*shard + + // I/O statistics for all shards. + stats Statistics + + // The read options to use for LevelDB. + readOpts *levigo.ReadOptions + + // The write options to use for LevelDB. + writeOpts *levigo.WriteOptions + + // If non-null, a channel we will send spans to once we finish writing them. This is only used + // for testing. + WrittenSpans chan *common.Span +} + +func CreateDataStore(cnf *conf.Config, writtenSpans chan *common.Span) (*dataStore, error) { + // Get the configuration. + clearStored := cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR) + dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES) + dirs := strings.Split(dirsStr, conf.PATH_LIST_SEP) + + var err error + lg := common.NewLogger("datastore", cnf) + store := &dataStore{lg: lg, shards: []*shard{}, WrittenSpans: writtenSpans} + + // If we return an error, close the store. + defer func() { + if err != nil { + store.Close() + store = nil + } + }() + + store.readOpts = levigo.NewReadOptions() + store.readOpts.SetFillCache(true) + store.writeOpts = levigo.NewWriteOptions() + store.writeOpts.SetSync(false) + + // Open all shards + for idx := range dirs { + path := dirs[idx] + conf.PATH_SEP + "db" + var shd *shard + shd, err = CreateShard(store, cnf, path, clearStored) + if err != nil { + lg.Errorf("Error creating shard %s: %s\n", path, err.Error()) + return nil, err + } + store.shards = append(store.shards, shd) + } + for idx := range store.shards { + shd := store.shards[idx] + shd.exited = make(chan bool, 1) + go shd.processIncoming() + } + return store, nil +} + +func CreateShard(store *dataStore, cnf *conf.Config, path string, + clearStored bool) (*shard, error) { + lg := store.lg + if clearStored { + fi, err := os.Stat(path) + if err != nil && !os.IsNotExist(err) { + lg.Errorf("Failed to stat %s: %s\n", path, err.Error()) + return nil, err + } + if fi != nil { + err = os.RemoveAll(path) + if err != nil { + lg.Errorf("Failed to clear existing datastore directory %s: %s\n", + path, err.Error()) + return nil, err + } + lg.Infof("Cleared existing datastore directory %s\n", path) + } + } + err := os.MkdirAll(path, 0777) + if err != nil { + lg.Errorf("Failed to MkdirAll(%s): %s\n", path, err.Error()) + return nil, err + } + var shd *shard + openOpts := levigo.NewOptions() + defer openOpts.Close() + newlyCreated := false + ldb, err := levigo.Open(path, openOpts) + if err == nil { + store.lg.Infof("LevelDB opened %s\n", path) + } else { + store.lg.Debugf("LevelDB failed to open %s: %s\n", path, err.Error()) + openOpts.SetCreateIfMissing(true) + ldb, err = levigo.Open(path, openOpts) + if err != nil { + store.lg.Errorf("LevelDB failed to create %s: %s\n", path, err.Error()) + return nil, err + } + store.lg.Infof("Created new LevelDB instance in %s\n", path) + newlyCreated = true + } + defer func() { + if shd == nil { + ldb.Close() + } + }() + lv, err := readLayoutVersion(store, ldb) + if err != nil { + store.lg.Errorf("Got error while reading datastore version for %s: %s\n", + path, err.Error()) + return nil, err + } + if newlyCreated && (lv == UNKNOWN_LAYOUT_VERSION) { + err = writeDataStoreVersion(store, ldb, CURRENT_LAYOUT_VERSION) + if err != nil { + store.lg.Errorf("Got error while writing datastore version for %s: %s\n", + path, err.Error()) + return nil, err + } + store.lg.Tracef("Wrote layout version %d to shard at %s.\n", + CURRENT_LAYOUT_VERSION, path) + } else if lv != CURRENT_LAYOUT_VERSION { + versionName := "unknown" + if lv != UNKNOWN_LAYOUT_VERSION { + versionName = fmt.Sprintf("%d", lv) + } + store.lg.Errorf("Can't read old datastore. Its layout version is %s, but this "+ + "software is at layout version %d. Please set %s to clear the datastore "+ + "on startup, or clear it manually.\n", versionName, + CURRENT_LAYOUT_VERSION, conf.HTRACE_DATA_STORE_CLEAR) + return nil, errors.New(fmt.Sprintf("Invalid layout version: got %s, expected %d.", + versionName, CURRENT_LAYOUT_VERSION)) + } else { + store.lg.Tracef("Found layout version %d in %s.\n", lv, path) + } + spanBufferSize := cnf.GetInt(conf.HTRACE_DATA_STORE_SPAN_BUFFER_SIZE) + shd = &shard{store: store, ldb: ldb, path: path, + incoming: make(chan *common.Span, spanBufferSize)} + return shd, nil +} + +// Read the datastore version of a leveldb instance. +func readLayoutVersion(store *dataStore, ldb *levigo.DB) (uint32, error) { + buf, err := ldb.Get(store.readOpts, []byte{VERSION_KEY}) + if err != nil { + return 0, err + } + if len(buf) == 0 { + return 0, nil + } + r := bytes.NewBuffer(buf) + decoder := gob.NewDecoder(r) + var v uint32 + err = decoder.Decode(&v) + if err != nil { + return 0, err + } + return v, nil +} + +// Write the datastore version to a shard. +func writeDataStoreVersion(store *dataStore, ldb *levigo.DB, v uint32) error { + w := new(bytes.Buffer) + encoder := gob.NewEncoder(w) + err := encoder.Encode(&v) + if err != nil { + return err + } + return ldb.Put(store.writeOpts, []byte{VERSION_KEY}, w.Bytes()) +} + +func (store *dataStore) GetStatistics() *Statistics { + return store.stats.Copy() +} + +// Close the DataStore. +func (store *dataStore) Close() { + for idx := range store.shards { + store.shards[idx].Close() + store.shards[idx] = nil + } + if store.readOpts != nil { + store.readOpts.Close() + store.readOpts = nil + } + if store.writeOpts != nil { + store.writeOpts.Close() + store.writeOpts = nil + } + if store.lg != nil { + store.lg.Close() + store.lg = nil + } +} + +// Get the index of the shard which stores the given spanId. +func (store *dataStore) getShardIndex(sid common.SpanId) int { + return int(sid.Val() % uint64(len(store.shards))) +} + +func (store *dataStore) WriteSpan(span *common.Span) { + store.shards[store.getShardIndex(span.Id)].incoming <- span +} + +func (store *dataStore) FindSpan(sid common.SpanId) *common.Span { + return store.shards[store.getShardIndex(sid)].FindSpan(sid) +} + +func (shd *shard) FindSpan(sid common.SpanId) *common.Span { + lg := shd.store.lg + buf, err := shd.ldb.Get(shd.store.readOpts, makeKey('s', sid.Val())) + if err != nil { + if strings.Index(err.Error(), "NotFound:") != -1 { + return nil + } + lg.Warnf("Shard(%s): FindSpan(%s) error: %s\n", + shd.path, sid.String(), err.Error()) + return nil + } + var span *common.Span + span, err = shd.decodeSpan(sid, buf) + if err != nil { + lg.Errorf("Shard(%s): FindSpan(%s) decode error: %s\n", + shd.path, sid.String(), err.Error()) + return nil + } + return span +} + +func (shd *shard) decodeSpan(sid common.SpanId, buf []byte) (*common.Span, error) { + r := bytes.NewBuffer(buf) + decoder := gob.NewDecoder(r) + data := common.SpanData{} + err := decoder.Decode(&data) + if err != nil { + return nil, err + } + // Gob encoding translates empty slices to nil. Reverse this so that we're always dealing with + // non-nil slices. + if data.Parents == nil { + data.Parents = []common.SpanId{} + } + return &common.Span{Id: common.SpanId(sid), SpanData: data}, nil +} + +// Find the children of a given span id. +func (store *dataStore) FindChildren(sid common.SpanId, lim int32) []common.SpanId { + childIds := make([]common.SpanId, 0) + var err error + + startIdx := store.getShardIndex(sid) + idx := startIdx + numShards := len(store.shards) + for { + if lim == 0 { + break + } + shd := store.shards[idx] + childIds, lim, err = shd.FindChildren(sid, childIds, lim) + if err != nil { + store.lg.Errorf("Shard(%s): FindChildren(%s) error: %s\n", + shd.path, sid.String(), err.Error()) + } + idx++ + if idx >= numShards { + idx = 0 + } + if idx == startIdx { + break + } + } + return childIds +} + +type predicateData struct { + *common.Predicate + uintKey uint64 + strKey string +} + +func loadPredicateData(pred *common.Predicate) (*predicateData, error) { + p := predicateData{Predicate: pred} + + // Parse the input value given to make sure it matches up with the field + // type. + switch pred.Field { + case common.SPAN_ID: + // Span IDs are sent as hex strings. + var id common.SpanId + if err := id.FromString(pred.Val); err != nil { + return nil, errors.New(fmt.Sprintf("Unable to parse span id '%s': %s", + pred.Val, err.Error())) + } + p.uintKey = id.Val() + break + case common.DESCRIPTION: + // Any string is valid for a description. + p.strKey = pred.Val + break + case common.BEGIN_TIME, common.END_TIME, common.DURATION: + // Parse a base-10 signed numeric field. + v, err := strconv.ParseInt(pred.Val, 10, 64) + if err != nil { + return nil, errors.New(fmt.Sprintf("Unable to parse %s '%s': %s", + pred.Field, pred.Val, err.Error())) + } + p.uintKey = s2u64(v) + break + default: + return nil, errors.New(fmt.Sprintf("Unknown field %s", pred.Field)) + } + + // Validate the predicate operation. + switch pred.Op { + case common.EQUALS, common.LESS_THAN_OR_EQUALS, + common.GREATER_THAN_OR_EQUALS, common.GREATER_THAN: + break + case common.CONTAINS: + if p.fieldIsNumeric() { + return nil, errors.New(fmt.Sprintf("Can't use CONTAINS on a "+ + "numeric field like '%s'", pred.Field)) + } + default: + return nil, errors.New(fmt.Sprintf("Unknown predicate operation '%s'", + pred.Op)) + } + + return &p, nil +} + +// Get the index prefix for this predicate, or 0 if it is not indexed. +func (pred *predicateData) getIndexPrefix() byte { + switch pred.Field { + case common.SPAN_ID: + return SPAN_ID_INDEX_PREFIX + case common.BEGIN_TIME: + return BEGIN_TIME_INDEX_PREFIX + case common.END_TIME: + return END_TIME_INDEX_PREFIX + case common.DURATION: + return DURATION_INDEX_PREFIX + default: + return INVALID_INDEX_PREFIX + } +} + +// Returns true if the predicate type is numeric. +func (pred *predicateData) fieldIsNumeric() bool { + switch pred.Field { + case common.SPAN_ID, common.BEGIN_TIME, common.END_TIME, common.DURATION: + return true + default: + return false + } +} + +// Get the values that this predicate cares about for a given span. +func (pred *predicateData) extractRelevantSpanData(span *common.Span) (uint64, string) { + switch pred.Field { + case common.SPAN_ID: + return span.Id.Val(), "" + case common.DESCRIPTION: + return 0, span.Description + case common.BEGIN_TIME: + return s2u64(span.Begin), "" + case common.END_TIME: + return s2u64(span.End), "" + case common.DURATION: + return s2u64(span.Duration()), "" + default: + panic(fmt.Sprintf("Field type %s isn't a 64-bit integer.", pred.Field)) + } +} + +func (pred *predicateData) spanPtrIsBefore(a *common.Span, b *common.Span) bool { + // nil is after everything. + if a == nil { + if b == nil { + return false + } + return false + } else if b == nil { + return true + } + // Compare the spans according to this predicate. + aInt, aStr := pred.extractRelevantSpanData(a) + bInt, bStr := pred.extractRelevantSpanData(b) + if pred.fieldIsNumeric() { + if pred.Op.IsDescending() { + return aInt > bInt + } else { + return aInt < bInt + } + } else { + if pred.Op.IsDescending() { + return aStr > bStr + } else { + return aStr < bStr + } + } +} + +// Returns true if the predicate is satisfied by the given span. +func (pred *predicateData) satisfiedBy(span *common.Span) bool { + intVal, strVal := pred.extractRelevantSpanData(span) + if pred.fieldIsNumeric() { + switch pred.Op { + case common.EQUALS: + return intVal == pred.uintKey + case common.LESS_THAN_OR_EQUALS: + return intVal <= pred.uintKey + case common.GREATER_THAN_OR_EQUALS: + return intVal >= pred.uintKey + case common.GREATER_THAN: + return intVal > pred.uintKey + default: + panic(fmt.Sprintf("unknown Op type %s should have been caught "+ + "during normalization", pred.Op)) + } + } else { + switch pred.Op { + case common.CONTAINS: + return strings.Contains(strVal, pred.strKey) + case common.EQUALS: + return strVal == pred.strKey + case common.LESS_THAN_OR_EQUALS: + return strVal <= pred.strKey + case common.GREATER_THAN_OR_EQUALS: + return strVal >= pred.strKey + case common.GREATER_THAN: + return strVal > pred.strKey + default: + panic(fmt.Sprintf("unknown Op type %s should have been caught "+ + "during normalization", pred.Op)) + } + } +} + +func (pred *predicateData) createSource(store *dataStore, prev *common.Span) (*source, error) { + var ret *source + src := source{store: store, + pred: pred, + iters: make([]*levigo.Iterator, 0, len(store.shards)), + nexts: make([]*common.Span, len(store.shards)), + numRead: make([]int, len(store.shards)), + keyPrefix: pred.getIndexPrefix(), + } + if src.keyPrefix == INVALID_INDEX_PREFIX { + return nil, errors.New(fmt.Sprintf("Can't create source from unindexed "+ + "predicate on field %s", pred.Field)) + } + defer func() { + if ret == nil { + src.Close() + } + }() + for shardIdx := range store.shards { + shd := store.shards[shardIdx] + src.iters = append(src.iters, shd.ldb.NewIterator(store.readOpts)) + } + var searchKey []byte + lg := store.lg + if prev != nil { + // If prev != nil, this query RPC is the continuation of a previous + // one. The final result returned the last time is 'prev'. + // + // To avoid returning the same results multiple times, we adjust the + // predicate here. If the predicate is on the span id field, we + // simply manipulate the span ID we're looking for. + // + // If the predicate is on a secondary index, we also use span ID, but + // in a slightly different way. Since the secondary indices are + // organized as [type-code][8b-secondary-key][8b-span-id], elements + // with the same secondary index field are ordered by span ID. So we + // create a 17-byte key incorporating the span ID from 'prev.' + var startId common.SpanId + switch pred.Op { + case common.EQUALS: + if pred.Field == common.SPAN_ID { + // This is an annoying corner case. There can only be one + // result each time we do an EQUALS search for a span id. + // Span id is the primary key for all our spans. + // But for some reason someone is asking for another result. + // We modify the query to search for the illegal 0 span ID, + // which will never be present. + lg.Debugf("Attempted to use a continuation token with an EQUALS "+ + "SPAN_ID query. %s. Setting search id = 0", + pred.Predicate.String()) + startId = 0 + } else { + // When doing an EQUALS search on a secondary index, the + // results are sorted by span id. + startId = prev.Id + 1 + } + case common.LESS_THAN_OR_EQUALS: + // Subtract one from the previous span id. Since the previous + // start ID will never be 0 (0 is an illegal span id), we'll never + // wrap around when doing this. + startId = prev.Id - 1 + case common.GREATER_THAN_OR_EQUALS: + // We can't add one to the span id, since the previous span ID + // might be the maximum value. So just switch over to using + // GREATER_THAN. + pred.Op = common.GREATER_THAN + startId = prev.Id + case common.GREATER_THAN: + // This one is easy. + startId = prev.Id + default: + str := fmt.Sprintf("Can't use a %v predicate as a source.", pred.Predicate.String()) + lg.Error(str + "\n") + panic(str) + } + if pred.Field == common.SPAN_ID { + pred.uintKey = uint64(startId) + searchKey = makeKey(src.keyPrefix, uint64(startId)) + } else { + // Start where the previous query left off. This means adjusting + // our uintKey. + pred.uintKey, _ = pred.extractRelevantSpanData(prev) + searchKey = makeSecondaryKey(src.keyPrefix, pred.uintKey, uint64(startId)) + } + if lg.TraceEnabled() { + lg.Tracef("Handling continuation token %s for %s. startId=%d, "+ + "pred.uintKey=%d\n", prev, pred.Predicate.String(), startId, + pred.uintKey) + } + } else { + searchKey = makeKey(src.keyPrefix, pred.uintKey) + } + for i := range src.iters { + src.iters[i].Seek(searchKey) + } + ret = &src + return ret, nil +} + +// A source of spans. +type source struct { + store *dataStore + pred *predicateData + iters []*levigo.Iterator + nexts []*common.Span + numRead []int + keyPrefix byte +} + +// Return true if this operation may require skipping the first result we get back from leveldb. +func mayRequireOneSkip(op common.Op) bool { + switch op { + // When dealing with descending predicates, the first span we read might not satisfy + // the predicate, even though subsequent ones will. This is because the iter.Seek() + // function "moves the iterator the position of the key given or, if the key doesn't + // exist, the next key that does exist in the database." So if we're on that "next + // key" it will not satisfy the predicate, but the keys previous to it might. + case common.LESS_THAN_OR_EQUALS: + return true + // iter.Seek basically takes us to the key which is "greater than or equal to" some + // value. Since we want greater than (not greater than or equal to) we may have to + // skip the first key. + case common.GREATER_THAN: + return true + } + return false +} + +// Fill in the entry in the 'next' array for a specific shard. +func (src *source) populateNextFromShard(shardIdx int) { + lg := src.store.lg + var err error + iter := src.iters[shardIdx] + if iter == nil { + lg.Debugf("Can't populate: No more entries in shard %d\n", shardIdx) + return // There are no more entries in this shard. + } + if src.nexts[shardIdx] != nil { + lg.Debugf("No need to populate shard %d\n", shardIdx) + return // We already have a valid entry for this shard. + } + for { + if !iter.Valid() { + lg.Debugf("Can't populate: Iterator for shard %d is no longer valid.\n", shardIdx) + break // Can't read past end of DB + } + src.numRead[shardIdx]++ + key := iter.Key() + if !bytes.HasPrefix(key, []byte{src.keyPrefix}) { + lg.Debugf("Can't populate: Iterator for shard %d does not have prefix %s\n", + shardIdx, string(src.keyPrefix)) + break // Can't read past end of indexed section + } + var span *common.Span + var sid common.SpanId + if src.keyPrefix == SPAN_ID_INDEX_PREFIX { + // The span id maps to the span itself. + sid = common.SpanId(keyToInt(key[1:])) + span, err = src.store.shards[shardIdx].decodeSpan(sid, iter.Value()) + if err != nil { + lg.Debugf("Internal error decoding span %s in shard %d: %s\n", + sid.String(), shardIdx, err.Error()) + break + } + } else { + // With a secondary index, we have to look up the span by id. + sid = common.SpanId(keyToInt(key[9:])) + span = src.store.shards[shardIdx].FindSpan(sid) + if span == nil { + lg.Debugf("Internal error rehydrating span %s in shard %d\n", + sid.String(), shardIdx) + break + } + } + if src.pred.Op.IsDescending() { + iter.Prev() + } else { + iter.Next() + } + if src.pred.satisfiedBy(span) { + lg.Debugf("Populated valid span %v from shard %d.\n", sid, shardIdx) + src.nexts[shardIdx] = span // Found valid entry + return + } else { + lg.Debugf("Span %s from shard %d does not satisfy the predicate.\n", + sid.String(), shardIdx) + if src.numRead[shardIdx] <= 1 && mayRequireOneSkip(src.pred.Op) { + continue + } + // This and subsequent entries don't satisfy predicate + break + } + } + lg.Debugf("Closing iterator for shard %d.\n", shardIdx) + iter.Close() + src.iters[shardIdx] = nil +} + +func (src *source) next() *common.Span { + for shardIdx := range src.iters { + src.populateNextFromShard(shardIdx) + } + var best *common.Span + bestIdx := -1 + for shardIdx := range src.iters { + span := src.nexts[shardIdx] + if src.pred.spanPtrIsBefore(span, best) { + best = span + bestIdx = shardIdx + } + } + if bestIdx >= 0 { + src.nexts[bestIdx] = nil + } + return best +} + +func (src *source) Close() { + for i := range src.iters { + if src.iters[i] != nil { + src.iters[i].Close() + } + } + src.iters = nil +} + +func (store *dataStore) obtainSource(preds *[]*predicateData, span *common.Span) (*source, error) { + // Read spans from the first predicate that is indexed. + p := *preds + for i := range p { + pred := p[i] + if pred.getIndexPrefix() != INVALID_INDEX_PREFIX { + *preds = append(p[0:i], p[i+1:]...) + return pred.createSource(store, span) + } + } + // If there are no predicates that are indexed, read rows in order of span id. + spanIdPred := common.Predicate{Op: common.GREATER_THAN_OR_EQUALS, + Field: common.SPAN_ID, + Val: "0000000000000000", + } + spanIdPredData, err := loadPredicateData(&spanIdPred) + if err != nil { + return nil, err + } + return spanIdPredData.createSource(store, span) +} + +func (store *dataStore) HandleQuery(query *common.Query) ([]*common.Span, error) { + lg := store.lg + // Parse predicate data. + var err error + preds := make([]*predicateData, len(query.Predicates)) + for i := range query.Predicates { + preds[i], err = loadPredicateData(&query.Predicates[i]) + if err != nil { + return nil, err + } + } + // Get a source of rows. + var src *source + src, err = store.obtainSource(&preds, query.Prev) + if err != nil { + return nil, err + } + defer src.Close() + lg.Debugf("HandleQuery %s: preds = %s, src = %v\n", query, preds, src) + + // Filter the spans through the remaining predicates. + ret := make([]*common.Span, 0, 32) + for { + if len(ret) >= query.Lim { + break // we hit the result size limit + } + span := src.next() + if span == nil { + break // the source has no more spans to give + } + if lg.DebugEnabled() { + lg.Debugf("src.next returned span %s\n", span.ToJson()) + } + satisfied := true + for predIdx := range preds { + if !preds[predIdx].satisfiedBy(span) { + satisfied = false + break + } + } + if satisfied { + ret = append(ret, span) + } + } + return ret, nil +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go new file mode 100644 index 0000000..4696547 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore_test.go @@ -0,0 +1,514 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bytes" + "encoding/json" + "math/rand" + htrace "org/apache/htrace/client" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "org/apache/htrace/test" + "os" + "sort" + "strings" + "testing" +) + +// Test creating and tearing down a datastore. +func TestCreateDatastore(t *testing.T) { + htraceBld := &MiniHTracedBuilder{Name: "TestCreateDatastore", + DataDirs: make([]string, 3)} + ht, err := htraceBld.Build() + if err != nil { + t.Fatalf("failed to create datastore: %s", err.Error()) + } + defer ht.Close() +} + +var SIMPLE_TEST_SPANS []common.Span = []common.Span{ + common.Span{Id: 1, + SpanData: common.SpanData{ + Begin: 123, + End: 456, + Description: "getFileDescriptors", + TraceId: 999, + Parents: []common.SpanId{}, + ProcessId: "firstd", + }}, + common.Span{Id: 2, + SpanData: common.SpanData{ + Begin: 125, + End: 200, + Description: "openFd", + TraceId: 999, + Parents: []common.SpanId{1}, + ProcessId: "secondd", + }}, + common.Span{Id: 3, + SpanData: common.SpanData{ + Begin: 200, + End: 456, + Description: "passFd", + TraceId: 999, + Parents: []common.SpanId{1}, + ProcessId: "thirdd", + }}, +} + +func createSpans(spans []common.Span, store *dataStore) { + for idx := range spans { + store.WriteSpan(&spans[idx]) + } + // Wait the spans to be created + for i := 0; i < 3; i++ { + <-store.WrittenSpans + } +} + +// Test creating a datastore and adding some spans. +func TestDatastoreWriteAndRead(t *testing.T) { + t.Parallel() + htraceBld := &MiniHTracedBuilder{Name: "TestDatastoreWriteAndRead", + WrittenSpans: make(chan *common.Span, 100)} + ht, err := htraceBld.Build() + if err != nil { + panic(err) + } + defer ht.Close() + createSpans(SIMPLE_TEST_SPANS, ht.Store) + if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { + t.Fatal() + } + span := ht.Store.FindSpan(1) + if span == nil { + t.Fatal() + } + if span.Id != 1 { + t.Fatal() + } + common.ExpectSpansEqual(t, &SIMPLE_TEST_SPANS[0], span) + children := ht.Store.FindChildren(1, 1) + if len(children) != 1 { + t.Fatalf("expected 1 child, but got %d\n", len(children)) + } + children = ht.Store.FindChildren(1, 2) + if len(children) != 2 { + t.Fatalf("expected 2 children, but got %d\n", len(children)) + } + sort.Sort(common.SpanIdSlice(children)) + if children[0] != 2 { + t.Fatal() + } + if children[1] != 3 { + t.Fatal() + } +} + +func testQuery(t *testing.T, ht *MiniHTraced, query *common.Query, + expectedSpans []common.Span) { + spans, err := ht.Store.HandleQuery(query) + if err != nil { + t.Fatalf("First query failed: %s\n", err.Error()) + } + expectedBuf := new(bytes.Buffer) + dec := json.NewEncoder(expectedBuf) + err = dec.Encode(expectedSpans) + if err != nil { + t.Fatalf("Failed to encode expectedSpans to JSON: %s\n", err.Error()) + } + spansBuf := new(bytes.Buffer) + dec = json.NewEncoder(spansBuf) + err = dec.Encode(spans) + if err != nil { + t.Fatalf("Failed to encode result spans to JSON: %s\n", err.Error()) + } + t.Logf("len(spans) = %d, len(expectedSpans) = %d\n", len(spans), + len(expectedSpans)) + common.ExpectStrEqual(t, string(expectedBuf.Bytes()), string(spansBuf.Bytes())) +} + +// Test queries on the datastore. +func TestSimpleQuery(t *testing.T) { + t.Parallel() + htraceBld := &MiniHTracedBuilder{Name: "TestSimpleQuery", + WrittenSpans: make(chan *common.Span, 100)} + ht, err := htraceBld.Build() + if err != nil { + panic(err) + } + defer ht.Close() + createSpans(SIMPLE_TEST_SPANS, ht.Store) + if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { + t.Fatal() + } + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.GREATER_THAN_OR_EQUALS, + Field: common.BEGIN_TIME, + Val: "125", + }, + }, + Lim: 5, + }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[2]}) +} + +func TestQueries2(t *testing.T) { + t.Parallel() + htraceBld := &MiniHTracedBuilder{Name: "TestQueries2", + WrittenSpans: make(chan *common.Span, 100)} + ht, err := htraceBld.Build() + if err != nil { + panic(err) + } + defer ht.Close() + createSpans(SIMPLE_TEST_SPANS, ht.Store) + if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { + t.Fatal() + } + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.LESS_THAN_OR_EQUALS, + Field: common.BEGIN_TIME, + Val: "125", + }, + }, + Lim: 5, + }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[0]}) + + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.LESS_THAN_OR_EQUALS, + Field: common.BEGIN_TIME, + Val: "125", + }, + common.Predicate{ + Op: common.EQUALS, + Field: common.DESCRIPTION, + Val: "getFileDescriptors", + }, + }, + Lim: 2, + }, []common.Span{SIMPLE_TEST_SPANS[0]}) + + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.EQUALS, + Field: common.DESCRIPTION, + Val: "getFileDescriptors", + }, + }, + Lim: 2, + }, []common.Span{SIMPLE_TEST_SPANS[0]}) +} + +func TestQueries3(t *testing.T) { + t.Parallel() + htraceBld := &MiniHTracedBuilder{Name: "TestQueries3", + WrittenSpans: make(chan *common.Span, 100)} + ht, err := htraceBld.Build() + if err != nil { + panic(err) + } + defer ht.Close() + createSpans(SIMPLE_TEST_SPANS, ht.Store) + if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { + t.Fatal() + } + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.CONTAINS, + Field: common.DESCRIPTION, + Val: "Fd", + }, + common.Predicate{ + Op: common.GREATER_THAN_OR_EQUALS, + Field: common.BEGIN_TIME, + Val: "100", + }, + }, + Lim: 5, + }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[2]}) + + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.LESS_THAN_OR_EQUALS, + Field: common.SPAN_ID, + Val: "0", + }, + }, + Lim: 200, + }, []common.Span{}) + + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.LESS_THAN_OR_EQUALS, + Field: common.SPAN_ID, + Val: "2", + }, + }, + Lim: 200, + }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[0]}) +} + +func TestQueries4(t *testing.T) { + t.Parallel() + htraceBld := &MiniHTracedBuilder{Name: "TestQueries4", + WrittenSpans: make(chan *common.Span, 100)} + ht, err := htraceBld.Build() + if err != nil { + panic(err) + } + defer ht.Close() + createSpans(SIMPLE_TEST_SPANS, ht.Store) + if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { + t.Fatal() + } + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.GREATER_THAN, + Field: common.BEGIN_TIME, + Val: "125", + }, + }, + Lim: 5, + }, []common.Span{SIMPLE_TEST_SPANS[2]}) + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.GREATER_THAN_OR_EQUALS, + Field: common.DESCRIPTION, + Val: "openFd", + }, + }, + Lim: 2, + }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[2]}) + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.GREATER_THAN, + Field: common.DESCRIPTION, + Val: "openFd", + }, + }, + Lim: 2, + }, []common.Span{SIMPLE_TEST_SPANS[2]}) +} + +func BenchmarkDatastoreWrites(b *testing.B) { + htraceBld := &MiniHTracedBuilder{Name: "BenchmarkDatastoreWrites", + WrittenSpans: make(chan *common.Span, b.N)} + ht, err := htraceBld.Build() + if err != nil { + panic(err) + } + defer ht.Close() + rnd := rand.New(rand.NewSource(1)) + allSpans := make([]*common.Span, b.N) + // Write many random spans. + for n := 0; n < b.N; n++ { + span := test.NewRandomSpan(rnd, allSpans[0:n]) + ht.Store.WriteSpan(span) + allSpans[n] = span + } + // Wait for all the spans to be written. + for n := 0; n < b.N; n++ { + <-ht.Store.WrittenSpans + } + spansWritten := ht.Store.GetStatistics().NumSpansWritten + if spansWritten < uint64(b.N) { + b.Fatal("incorrect statistics: expected %d spans to be written, but only got %d", + b.N, spansWritten) + } +} + +func TestReloadDataStore(t *testing.T) { + htraceBld := &MiniHTracedBuilder{Name: "TestReloadDataStore", + DataDirs: make([]string, 2), KeepDataDirsOnClose: true} + ht, err := htraceBld.Build() + if err != nil { + t.Fatalf("failed to create datastore: %s", err.Error()) + } + dataDirs := make([]string, len(ht.DataDirs)) + copy(dataDirs, ht.DataDirs) + defer func() { + if ht != nil { + ht.Close() + } + for i := range dataDirs { + os.RemoveAll(dataDirs[i]) + } + }() + var hcl *htrace.Client + hcl, err = htrace.NewClient(ht.ClientConf()) + if err != nil { + t.Fatalf("failed to create client: %s", err.Error()) + } + + // Create some random trace spans. + NUM_TEST_SPANS := 5 + allSpans := createRandomTestSpans(NUM_TEST_SPANS) + err = hcl.WriteSpans(&common.WriteSpansReq{ + Spans: allSpans, + }) + if err != nil { + t.Fatalf("WriteSpans failed: %s\n", err.Error()) + } + + // Look up the spans we wrote. + var span *common.Span + for i := 0; i < NUM_TEST_SPANS; i++ { + span, err = hcl.FindSpan(allSpans[i].Id) + if err != nil { + t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) + } + common.ExpectSpansEqual(t, allSpans[i], span) + } + + ht.Close() + ht = nil + + htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore2", + DataDirs: dataDirs, KeepDataDirsOnClose: true} + ht, err = htraceBld.Build() + if err != nil { + t.Fatalf("failed to re-create datastore: %s", err.Error()) + } + hcl, err = htrace.NewClient(ht.ClientConf()) + if err != nil { + t.Fatalf("failed to re-create client: %s", err.Error()) + } + + // Look up the spans we wrote earlier. + for i := 0; i < NUM_TEST_SPANS; i++ { + span, err = hcl.FindSpan(allSpans[i].Id) + if err != nil { + t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) + } + common.ExpectSpansEqual(t, allSpans[i], span) + } + + // Set an old datastore version number. + for i := range ht.Store.shards { + shard := ht.Store.shards[i] + writeDataStoreVersion(ht.Store, shard.ldb, CURRENT_LAYOUT_VERSION-1) + } + ht.Close() + ht = nil + + htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore3", + DataDirs: dataDirs, KeepDataDirsOnClose: true} + ht, err = htraceBld.Build() + if err == nil { + t.Fatalf("expected the datastore to fail to load after setting an " + + "incorrect version.\n") + } + if !strings.Contains(err.Error(), "Invalid layout version") { + t.Fatal(`expected the loading error to contain "invalid layout version"` + "\n") + } + + // It should work with data.store.clear set. + htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore4", + DataDirs: dataDirs, KeepDataDirsOnClose: true, + Cnf: map[string]string{conf.HTRACE_DATA_STORE_CLEAR: "true"}} + ht, err = htraceBld.Build() + if err != nil { + t.Fatalf("expected the datastore loading to succeed after setting an "+ + "incorrect version. But it failed with error %s\n", err.Error()) + } +} + +func TestQueriesWithContinuationTokens1(t *testing.T) { + t.Parallel() + htraceBld := &MiniHTracedBuilder{Name: "TestQueriesWithContinuationTokens1", + WrittenSpans: make(chan *common.Span, 100)} + ht, err := htraceBld.Build() + if err != nil { + panic(err) + } + defer ht.Close() + createSpans(SIMPLE_TEST_SPANS, ht.Store) + if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { + t.Fatal() + } + // Adding a prev value to this query excludes the first result that we + // would normally get. + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.GREATER_THAN, + Field: common.BEGIN_TIME, + Val: "120", + }, + }, + Lim: 5, + Prev: &SIMPLE_TEST_SPANS[0], + }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[2]}) + + // There is only one result from an EQUALS query on SPAN_ID. + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.EQUALS, + Field: common.SPAN_ID, + Val: "1", + }, + }, + Lim: 100, + Prev: &SIMPLE_TEST_SPANS[0], + }, []common.Span{}) + + // When doing a LESS_THAN_OR_EQUALS search, we still don't get back the + // span we pass as a continuation token. (Primary index edition). + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.LESS_THAN_OR_EQUALS, + Field: common.SPAN_ID, + Val: "2", + }, + }, + Lim: 100, + Prev: &SIMPLE_TEST_SPANS[1], + }, []common.Span{SIMPLE_TEST_SPANS[0]}) + + // When doing a GREATER_THAN_OR_EQUALS search, we still don't get back the + // span we pass as a continuation token. (Secondary index edition). + testQuery(t, ht, &common.Query{ + Predicates: []common.Predicate{ + common.Predicate{ + Op: common.GREATER_THAN, + Field: common.DURATION, + Val: "0", + }, + }, + Lim: 100, + Prev: &SIMPLE_TEST_SPANS[1], + }, []common.Span{SIMPLE_TEST_SPANS[2], SIMPLE_TEST_SPANS[0]}) +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go b/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go new file mode 100644 index 0000000..a53380e --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/hrpc.go @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bufio" + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "github.com/ugorji/go/codec" + "io" + "net" + "net/rpc" + "org/apache/htrace/common" + "org/apache/htrace/conf" +) + +// Handles HRPC calls +type HrpcHandler struct { + lg *common.Logger + store *dataStore +} + +// The HRPC server +type HrpcServer struct { + *rpc.Server + hand *HrpcHandler + listener net.Listener +} + +// Codec which encodes HRPC data via JSON +type HrpcServerCodec struct { + lg *common.Logger + conn net.Conn + length uint32 +} + +func asJson(val interface{}) string { + js, err := json.Marshal(val) + if err != nil { + return "encoding error: " + err.Error() + } + return string(js) +} + +func createErrAndWarn(lg *common.Logger, val string) error { + return createErrAndLog(lg, val, common.WARN) +} + +func createErrAndLog(lg *common.Logger, val string, level common.Level) error { + lg.Write(level, val+"\n") + return errors.New(val) +} + +func (cdc *HrpcServerCodec) ReadRequestHeader(req *rpc.Request) error { + hdr := common.HrpcRequestHeader{} + if cdc.lg.TraceEnabled() { + cdc.lg.Tracef("Reading HRPC request header from %s\n", cdc.conn.RemoteAddr()) + } + err := binary.Read(cdc.conn, binary.LittleEndian, &hdr) + if err != nil { + level := common.WARN + if err == io.EOF { + level = common.DEBUG + } + return createErrAndLog(cdc.lg, fmt.Sprintf("Error reading header bytes: %s", + err.Error()), level) + } + if cdc.lg.TraceEnabled() { + cdc.lg.Tracef("Read HRPC request header %s from %s\n", + asJson(&hdr), cdc.conn.RemoteAddr()) + } + if hdr.Magic != common.HRPC_MAGIC { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Invalid request header: expected "+ + "magic number of 0x%04x, but got 0x%04x", common.HRPC_MAGIC, hdr.Magic)) + } + if hdr.Length > common.MAX_HRPC_BODY_LENGTH { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Length prefix was too long. "+ + "Maximum length is %d, but we got %d.", common.MAX_HRPC_BODY_LENGTH, + hdr.Length)) + } + req.ServiceMethod = common.HrpcMethodIdToMethodName(hdr.MethodId) + if req.ServiceMethod == "" { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Unknown MethodID code 0x%04x", + hdr.MethodId)) + } + req.Seq = hdr.Seq + cdc.length = hdr.Length + return nil +} + +func (cdc *HrpcServerCodec) ReadRequestBody(body interface{}) error { + if cdc.lg.TraceEnabled() { + cdc.lg.Tracef("Reading HRPC %d-byte request body from %s\n", + cdc.length, cdc.conn.RemoteAddr()) + } + mh := new(codec.MsgpackHandle) + mh.WriteExt = true + dec := codec.NewDecoder(io.LimitReader(cdc.conn, int64(cdc.length)), mh) + err := dec.Decode(body) + if err != nil { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to read request "+ + "body from %s: %s", cdc.conn.RemoteAddr(), err.Error())) + } + if cdc.lg.TraceEnabled() { + cdc.lg.Tracef("Read body from %s: %s\n", + cdc.conn.RemoteAddr(), asJson(&body)) + } + return nil +} + +var EMPTY []byte = make([]byte, 0) + +func (cdc *HrpcServerCodec) WriteResponse(resp *rpc.Response, msg interface{}) error { + var err error + buf := EMPTY + if msg != nil { + mh := new(codec.MsgpackHandle) + mh.WriteExt = true + w := bytes.NewBuffer(make([]byte, 0, 128)) + enc := codec.NewEncoder(w, mh) + err := enc.Encode(msg) + if err != nil { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to marshal "+ + "response message: %s", err.Error())) + } + buf = w.Bytes() + } + hdr := common.HrpcResponseHeader{} + hdr.MethodId = common.HrpcMethodNameToId(resp.ServiceMethod) + hdr.Seq = resp.Seq + hdr.ErrLength = uint32(len(resp.Error)) + hdr.Length = uint32(len(buf)) + writer := bufio.NewWriterSize(cdc.conn, 256) + err = binary.Write(writer, binary.LittleEndian, &hdr) + if err != nil { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write response "+ + "header: %s", err.Error())) + } + if hdr.ErrLength > 0 { + _, err = io.WriteString(writer, resp.Error) + if err != nil { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write error "+ + "string: %s", err.Error())) + } + } + if hdr.Length > 0 { + var length int + length, err = writer.Write(buf) + if err != nil { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write response "+ + "message: %s", err.Error())) + } + if uint32(length) != hdr.Length { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write all of "+ + "response message: %s", err.Error())) + } + } + err = writer.Flush() + if err != nil { + return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write the response "+ + "bytes: %s", err.Error())) + } + return nil +} + +func (cdc *HrpcServerCodec) Close() error { + return cdc.conn.Close() +} + +func (hand *HrpcHandler) WriteSpans(req *common.WriteSpansReq, + resp *common.WriteSpansResp) (err error) { + hand.lg.Debugf("hrpc writeSpansHandler: received %d span(s). "+ + "defaultPid = %s\n", len(req.Spans), req.DefaultPid) + for i := range req.Spans { + span := req.Spans[i] + if span.ProcessId == "" { + span.ProcessId = req.DefaultPid + } + if hand.lg.TraceEnabled() { + hand.lg.Tracef("writing span %d: %s\n", i, span.ToJson()) + } + hand.store.WriteSpan(span) + } + return nil +} + +func CreateHrpcServer(cnf *conf.Config, store *dataStore) (*HrpcServer, error) { + lg := common.NewLogger("hrpc", cnf) + hsv := &HrpcServer{ + Server: rpc.NewServer(), + hand: &HrpcHandler{ + lg: lg, + store: store, + }, + } + var err error + hsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_HRPC_ADDRESS)) + if err != nil { + return nil, err + } + hsv.Server.Register(hsv.hand) + go hsv.run() + lg.Infof("Started HRPC server on %s...\n", hsv.listener.Addr().String()) + return hsv, nil +} + +func (hsv *HrpcServer) run() { + lg := hsv.hand.lg + for { + conn, err := hsv.listener.Accept() + if err != nil { + lg.Errorf("HRPC Accept error: %s\n", err.Error()) + continue + } + if lg.TraceEnabled() { + lg.Tracef("Accepted HRPC connection from %s\n", conn.RemoteAddr()) + } + go hsv.ServeCodec(&HrpcServerCodec{ + lg: lg, + conn: conn, + }) + } +} + +func (hsv *HrpcServer) Addr() net.Addr { + return hsv.listener.Addr() +} + +func (hsv *HrpcServer) Close() { + hsv.listener.Close() +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/htraced.go b/htrace-htraced/go/src/org/apache/htrace/htraced/htraced.go new file mode 100644 index 0000000..64da457 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/htraced.go @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "encoding/json" + "fmt" + "net" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "os" + "strings" + "time" +) + +var RELEASE_VERSION string +var GIT_VERSION string + +const USAGE = `htraced: the HTrace server daemon. + +htraced receives trace spans sent from HTrace clients. It exposes a REST +interface which others can query. It also runs a web server with a graphical +user interface. htraced stores its span data in levelDB files on the local +disks. + +Usage: +--help: this help message + +-Dk=v: set configuration key 'k' to value 'v' +For example -Dweb.address=127.0.0.1:8080 sets the web address to localhost, +port 8080. + +-Dk: set configuration key 'k' to 'true' + +Normally, configuration options should be set in the ` + conf.CONFIG_FILE_NAME + ` +configuration file. We find this file by searching the paths in the +` + conf.HTRACED_CONF_DIR + `. The command-line options are just an alternate way +of setting configuration when launching the daemon. +` + +func main() { + for idx := range os.Args { + arg := os.Args[idx] + if strings.HasPrefix(arg, "--h") || strings.HasPrefix(arg, "-h") { + fmt.Fprintf(os.Stderr, USAGE) + os.Exit(0) + } + } + cnf := common.LoadApplicationConfig() + common.InstallSignalHandlers(cnf) + lg := common.NewLogger("main", cnf) + defer lg.Close() + store, err := CreateDataStore(cnf, nil) + if err != nil { + lg.Errorf("Error creating datastore: %s\n", err.Error()) + os.Exit(1) + } + var rsv *RestServer + rsv, err = CreateRestServer(cnf, store) + if err != nil { + lg.Errorf("Error creating REST server: %s\n", err.Error()) + os.Exit(1) + } + var hsv *HrpcServer + if cnf.Get(conf.HTRACE_HRPC_ADDRESS) != "" { + hsv, err = CreateHrpcServer(cnf, store) + if err != nil { + lg.Errorf("Error creating HRPC server: %s\n", err.Error()) + os.Exit(1) + } + } else { + lg.Infof("Not starting HRPC server because no value was given for %s.\n", + conf.HTRACE_HRPC_ADDRESS) + } + naddr := cnf.Get(conf.HTRACE_STARTUP_NOTIFICATION_ADDRESS) + if naddr != "" { + notif := StartupNotification{ + HttpAddr: rsv.Addr().String(), + ProcessId: os.Getpid(), + } + if hsv != nil { + notif.HrpcAddr = hsv.Addr().String() + } + err = sendStartupNotification(naddr, ¬if) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to send startup notification: "+ + "%s\n", err.Error()) + os.Exit(1) + } + } + for { + time.Sleep(time.Duration(10) * time.Hour) + } +} + +// A startup notification message that we optionally send on startup. +// Used by unit tests. +type StartupNotification struct { + HttpAddr string + HrpcAddr string + ProcessId int +} + +func sendStartupNotification(naddr string, notif *StartupNotification) error { + conn, err := net.Dial("tcp", naddr) + if err != nil { + return err + } + defer func() { + if conn != nil { + conn.Close() + } + }() + var buf []byte + buf, err = json.Marshal(notif) + if err != nil { + return err + } + _, err = conn.Write(buf) + conn.Close() + conn = nil + return nil +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/mini_htraced.go b/htrace-htraced/go/src/org/apache/htrace/htraced/mini_htraced.go new file mode 100644 index 0000000..a54f2cb --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/mini_htraced.go @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "fmt" + "io/ioutil" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "os" + "strings" +) + +// +// MiniHTraceD is used in unit tests to set up a daemon with certain settings. +// It takes care of things like creating and cleaning up temporary directories. +// + +// The default number of managed data directories to use. +const DEFAULT_NUM_DATA_DIRS = 2 + +// Builds a MiniHTraced object. +type MiniHTracedBuilder struct { + // The name of the MiniHTraced to build. This shows up in the test directory name and some + // other places. + Name string + + // The configuration values to use for the MiniHTraced. + // If ths is nil, we use the default configuration for everything. + Cnf map[string]string + + // The DataDirs to use. Empty entries will turn into random names. + DataDirs []string + + // If true, we will keep the data dirs around after MiniHTraced#Close + KeepDataDirsOnClose bool + + // If non-null, the WrittenSpans channel to use when creating the DataStore. + WrittenSpans chan *common.Span +} + +type MiniHTraced struct { + Name string + Cnf *conf.Config + DataDirs []string + Store *dataStore + Rsv *RestServer + Hsv *HrpcServer + Lg *common.Logger + KeepDataDirsOnClose bool +} + +func (bld *MiniHTracedBuilder) Build() (*MiniHTraced, error) { + var err error + var store *dataStore + var rsv *RestServer + var hsv *HrpcServer + if bld.Name == "" { + bld.Name = "HTraceTest" + } + if bld.Cnf == nil { + bld.Cnf = make(map[string]string) + } + if bld.DataDirs == nil { + bld.DataDirs = make([]string, 2) + } + for idx := range bld.DataDirs { + if bld.DataDirs[idx] == "" { + bld.DataDirs[idx], err = ioutil.TempDir(os.TempDir(), + fmt.Sprintf("%s%d", bld.Name, idx+1)) + if err != nil { + return nil, err + } + } + } + bld.Cnf[conf.HTRACE_DATA_STORE_DIRECTORIES] = + strings.Join(bld.DataDirs, conf.PATH_LIST_SEP) + bld.Cnf[conf.HTRACE_WEB_ADDRESS] = ":0" // use a random port for the REST server + bld.Cnf[conf.HTRACE_HRPC_ADDRESS] = ":0" // use a random port for the HRPC server + bld.Cnf[conf.HTRACE_LOG_LEVEL] = "TRACE" + cnfBld := conf.Builder{Values: bld.Cnf, Defaults: conf.DEFAULTS} + cnf, err := cnfBld.Build() + if err != nil { + return nil, err + } + lg := common.NewLogger("mini.htraced", cnf) + defer func() { + if err != nil { + if store != nil { + store.Close() + } + for idx := range bld.DataDirs { + if bld.DataDirs[idx] != "" { + os.RemoveAll(bld.DataDirs[idx]) + } + } + if rsv != nil { + rsv.Close() + } + lg.Infof("Failed to create MiniHTraced %s: %s\n", bld.Name, err.Error()) + lg.Close() + } + }() + store, err = CreateDataStore(cnf, bld.WrittenSpans) + if err != nil { + return nil, err + } + rsv, err = CreateRestServer(cnf, store) + if err != nil { + return nil, err + } + hsv, err = CreateHrpcServer(cnf, store) + if err != nil { + return nil, err + } + + lg.Infof("Created MiniHTraced %s\n", bld.Name) + return &MiniHTraced{ + Name: bld.Name, + Cnf: cnf, + DataDirs: bld.DataDirs, + Store: store, + Rsv: rsv, + Hsv: hsv, + Lg: lg, + KeepDataDirsOnClose: bld.KeepDataDirsOnClose, + }, nil +} + +// Return a Config object that clients can use to connect to this MiniHTraceD. +func (ht *MiniHTraced) ClientConf() *conf.Config { + return ht.Cnf.Clone(conf.HTRACE_WEB_ADDRESS, ht.Rsv.Addr().String(), + conf.HTRACE_HRPC_ADDRESS, ht.Hsv.Addr().String()) +} + +func (ht *MiniHTraced) Close() { + ht.Lg.Infof("Closing MiniHTraced %s\n", ht.Name) + ht.Rsv.Close() + ht.Store.Close() + if !ht.KeepDataDirsOnClose { + for idx := range ht.DataDirs { + ht.Lg.Infof("Removing %s...\n", ht.DataDirs[idx]) + os.RemoveAll(ht.DataDirs[idx]) + } + } + ht.Lg.Infof("Finished closing MiniHTraced %s\n", ht.Name) + ht.Lg.Close() +} diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go b/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go new file mode 100644 index 0000000..69b316c --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/rest.go @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/gorilla/mux" + "io" + "net" + "net/http" + "org/apache/htrace/common" + "org/apache/htrace/conf" + "os" + "path/filepath" + "strconv" + "strings" +) + +// Set the response headers. +func setResponseHeaders(hdr http.Header) { + hdr.Set("Content-Type", "application/json") +} + +// Write a JSON error response. +func writeError(lg *common.Logger, w http.ResponseWriter, errCode int, + errStr string) { + str := strings.Replace(errStr, `"`, `'`, -1) + lg.Info(str + "\n") + w.WriteHeader(errCode) + w.Write([]byte(`{ "error" : "` + str + `"}`)) +} + +type serverInfoHandler struct { + lg *common.Logger +} + +func (hand *serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + setResponseHeaders(w.Header()) + version := common.ServerInfo{ReleaseVersion: RELEASE_VERSION, + GitVersion: GIT_VERSION} + buf, err := json.Marshal(&version) + if err != nil { + writeError(hand.lg, w, http.StatusInternalServerError, + fmt.Sprintf("error marshalling ServerInfo: %s\n", err.Error())) + return + } + if hand.lg.DebugEnabled() { + hand.lg.Debugf("Returned serverInfo %s\n", string(buf)) + } + w.Write(buf) +} + +type dataStoreHandler struct { + lg *common.Logger + store *dataStore +} + +func (hand *dataStoreHandler) parseSid(w http.ResponseWriter, + str string) (common.SpanId, bool) { + val, err := strconv.ParseUint(str, 16, 64) + if err != nil { + writeError(hand.lg, w, http.StatusBadRequest, + fmt.Sprintf("Failed to parse span ID %s: %s", str, err.Error())) + w.Write([]byte("Error parsing : " + err.Error())) + return 0, false + } + return common.SpanId(val), true +} + +func (hand *dataStoreHandler) getReqField32(fieldName string, w http.ResponseWriter, + req *http.Request) (int32, bool) { + str := req.FormValue(fieldName) + if str == "" { + writeError(hand.lg, w, http.StatusBadRequest, fmt.Sprintf("No %s specified.", fieldName)) + return -1, false + } + val, err := strconv.ParseUint(str, 16, 32) + if err != nil { + writeError(hand.lg, w, http.StatusBadRequest, + fmt.Sprintf("Error parsing %s: %s.", fieldName, err.Error())) + return -1, false + } + return int32(val), true +} + +type findSidHandler struct { + dataStoreHandler +} + +func (hand *findSidHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + setResponseHeaders(w.Header()) + req.ParseForm() + vars := mux.Vars(req) + stringSid := vars["id"] + sid, ok := hand.parseSid(w, stringSid) + if !ok { + return + } + hand.lg.Debugf("findSidHandler(sid=%s)\n", sid.String()) + span := hand.store.FindSpan(sid) + if span == nil { + writeError(hand.lg, w, http.StatusNoContent, + fmt.Sprintf("No such span as %s\n", sid.String())) + return + } + w.Write(span.ToJson()) +} + +type findChildrenHandler struct { + dataStoreHandler +} + +func (hand *findChildrenHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + setResponseHeaders(w.Header()) + req.ParseForm() + vars := mux.Vars(req) + stringSid := vars["id"] + sid, ok := hand.parseSid(w, stringSid) + if !ok { + return + } + var lim int32 + lim, ok = hand.getReqField32("lim", w, req) + if !ok { + return + } + hand.lg.Debugf("findChildrenHandler(sid=%s, lim=%d)\n", sid.String(), lim) + children := hand.store.FindChildren(sid, lim) + jbytes, err := json.Marshal(children) + if err != nil { + writeError(hand.lg, w, http.StatusInternalServerError, + fmt.Sprintf("Error marshalling children: %s", err.Error())) + return + } + w.Write(jbytes) +} + +type writeSpansHandler struct { + dataStoreHandler +} + +func (hand *writeSpansHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + setResponseHeaders(w.Header()) + dec := json.NewDecoder(req.Body) + spans := make([]*common.Span, 0, 32) + defaultPid := req.Header.Get("htrace-pid") + for { + var span common.Span + err := dec.Decode(&span) + if err != nil { + if err != io.EOF { + writeError(hand.lg, w, http.StatusBadRequest, + fmt.Sprintf("Error parsing spans: %s", err.Error())) + return + } + break + } + if span.ProcessId == "" { + span.ProcessId = defaultPid + } + spans = append(spans, &span) + } + hand.lg.Debugf("writeSpansHandler: received %d span(s). defaultPid = %s\n", + len(spans), defaultPid) + for spanIdx := range spans { + if hand.lg.DebugEnabled() { + hand.lg.Debugf("writing span %s\n", spans[spanIdx].ToJson()) + } + hand.store.WriteSpan(spans[spanIdx]) + } +} + +type queryHandler struct { + lg *common.Logger + dataStoreHandler +} + +func (hand *queryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + setResponseHeaders(w.Header()) + queryString := req.FormValue("query") + if queryString == "" { + writeError(hand.lg, w, http.StatusBadRequest, "No query provided.\n") + return + } + var query common.Query + reader := bytes.NewBufferString(queryString) + dec := json.NewDecoder(reader) + err := dec.Decode(&query) + if err != nil { + writeError(hand.lg, w, http.StatusBadRequest, + fmt.Sprintf("Error parsing query: %s", err.Error())) + return + } + var results []*common.Span + results, err = hand.store.HandleQuery(&query) + if err != nil { + writeError(hand.lg, w, http.StatusInternalServerError, + fmt.Sprintf("Internal error processing query %s: %s", + query.String(), err.Error())) + return + } + var jbytes []byte + jbytes, err = json.Marshal(results) + if err != nil { + writeError(hand.lg, w, http.StatusInternalServerError, + fmt.Sprintf("Error marshalling results: %s", err.Error())) + return + } + w.Write(jbytes) +} + +type logErrorHandler struct { + lg *common.Logger +} + +func (hand *logErrorHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + hand.lg.Errorf("Got unknown request %s\n", req.RequestURI) + writeError(hand.lg, w, http.StatusBadRequest, "Unknown request.") +} + +type RestServer struct { + listener net.Listener + lg *common.Logger +} + +func CreateRestServer(cnf *conf.Config, store *dataStore) (*RestServer, error) { + var err error + rsv := &RestServer{} + rsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_WEB_ADDRESS)) + if err != nil { + return nil, err + } + var success bool + defer func() { + if !success { + rsv.Close() + } + }() + rsv.lg = common.NewLogger("rest", cnf) + + r := mux.NewRouter().StrictSlash(false) + + r.Handle("/server/info", &serverInfoHandler{lg: rsv.lg}).Methods("GET") + + writeSpansH := &writeSpansHandler{dataStoreHandler: dataStoreHandler{ + store: store, lg: rsv.lg}} + r.Handle("/writeSpans", writeSpansH).Methods("POST") + + queryH := &queryHandler{lg: rsv.lg, dataStoreHandler: dataStoreHandler{store: store}} + r.Handle("/query", queryH).Methods("GET") + + span := r.PathPrefix("/span").Subrouter() + findSidH := &findSidHandler{dataStoreHandler: dataStoreHandler{store: store, lg: rsv.lg}} + span.Handle("/{id}", findSidH).Methods("GET") + + findChildrenH := &findChildrenHandler{dataStoreHandler: dataStoreHandler{store: store, + lg: rsv.lg}} + span.Handle("/{id}/children", findChildrenH).Methods("GET") + + // Default Handler. This will serve requests for static requests. + webdir := os.Getenv("HTRACED_WEB_DIR") + if webdir == "" { + webdir, err = filepath.Abs(filepath.Join(filepath.Dir(os.Args[0]), "..", "..", "web")) + + if err != nil { + return nil, err + } + } + + rsv.lg.Infof("Serving static files from %s\n.", webdir) + r.PathPrefix("/").Handler(http.FileServer(http.Dir(webdir))).Methods("GET") + + // Log an error message for unknown non-GET requests. + r.PathPrefix("/").Handler(&logErrorHandler{lg: rsv.lg}) + + go http.Serve(rsv.listener, r) + + rsv.lg.Infof("Started REST server on %s...\n", rsv.listener.Addr().String()) + success = true + return rsv, nil +} + +func (rsv *RestServer) Addr() net.Addr { + return rsv.listener.Addr() +} + +func (rsv *RestServer) Close() { + rsv.listener.Close() +} diff --git a/htrace-htraced/go/src/org/apache/htrace/test/random.go b/htrace-htraced/go/src/org/apache/htrace/test/random.go new file mode 100644 index 0000000..d10e2f9 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/test/random.go @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package test + +import ( + "fmt" + "math/rand" + "org/apache/htrace/common" +) + +func NonZeroRand64(rnd *rand.Rand) int64 { + for { + r := rnd.Int63() + if r == 0 { + continue + } + if rnd.Intn(1) != 0 { + return -r + } + return r + } +} + +func NonZeroRand32(rnd *rand.Rand) int32 { + for { + r := rnd.Int31() + if r == 0 { + continue + } + if rnd.Intn(1) != 0 { + return -r + } + return r + } +} + +// Create a random span. +func NewRandomSpan(rnd *rand.Rand, potentialParents []*common.Span) *common.Span { + parents := []common.SpanId{} + if potentialParents != nil { + parentIdx := rnd.Intn(len(potentialParents) + 1) + if parentIdx < len(potentialParents) { + parents = []common.SpanId{potentialParents[parentIdx].Id} + } + } + return &common.Span{Id: common.SpanId(NonZeroRand64(rnd)), + SpanData: common.SpanData{ + Begin: NonZeroRand64(rnd), + End: NonZeroRand64(rnd), + Description: "getFileDescriptors", + TraceId: common.SpanId(NonZeroRand64(rnd)), + Parents: parents, + ProcessId: fmt.Sprintf("process%d", NonZeroRand32(rnd)), + }} +} diff --git a/htrace-htraced/go/src/org/apache/htrace/test/util.go b/htrace-htraced/go/src/org/apache/htrace/test/util.go new file mode 100644 index 0000000..cc058e0 --- /dev/null +++ b/htrace-htraced/go/src/org/apache/htrace/test/util.go @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package test + +import ( + "org/apache/htrace/common" +) + +func SpanId(str string) common.SpanId { + var spanId common.SpanId + err := spanId.FromString(str) + if err != nil { + panic(err.Error()) + } + return spanId +} diff --git a/htrace-htraced/pom.xml b/htrace-htraced/pom.xml index 94c2dba..a56a79d 100644 --- a/htrace-htraced/pom.xml +++ b/htrace-htraced/pom.xml @@ -109,9 +109,9 @@ language governing permissions and limitations under the License. --> run - + @@ -123,7 +123,7 @@ language governing permissions and limitations under the License. --> run - + @@ -137,7 +137,7 @@ language governing permissions and limitations under the License. --> run - + @@ -203,7 +203,7 @@ language governing permissions and limitations under the License. --> This is new-style jetty client, jetty9 and jdk7 required. It can do async but we will use it synchronously at first. Has nice tutorial: http://www.eclipse.org/jetty/documentation/9.1.5.v20140505/http-client-api.html - --> + --> org.eclipse.jetty jetty-client @@ -249,10 +249,10 @@ language governing permissions and limitations under the License. --> 755 - src/go/build/htrace + go/build/htrace - src/go/build/htraced + go/build/htraced diff --git a/htrace-htraced/src/go/Godeps/Godeps.json b/htrace-htraced/src/go/Godeps/Godeps.json deleted file mode 100644 index 47aa90e..0000000 --- a/htrace-htraced/src/go/Godeps/Godeps.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "ImportPath": "git-wip-us.apache.org/repos/asf/incubator-htrace.git", - "GoVersion": "go1.3.1", - "Deps": [ - { - "ImportPath": "github.com/alecthomas/kingpin", - "Rev": "afafa8aab106d31c9dc8f5e562b3f30f6246c3d4" - }, - { - "ImportPath": "github.com/alecthomas/units", - "Rev": "6b4e7dc5e3143b85ea77909c72caf89416fc2915" - }, - { - "ImportPath": "github.com/gorilla/context", - "Rev": "215affda49addc4c8ef7e2534915df2c8c35c6cd" - }, - { - "ImportPath": "github.com/gorilla/mux", - "Rev": "e444e69cbd2e2e3e0749a2f3c717cec491552bbf" - }, - { - "ImportPath": "github.com/jmhodges/levigo", - "Rev": "2c43dde93d0e056173706534afd514fcbc1dd578" - }, - { - "ImportPath": "github.com/ugorji/go/codec", - "Rev": "08bbe4aa39b9f189f4e294b5c8408b5fa5787bb2" - } - ] -} diff --git a/htrace-htraced/src/go/format.sh b/htrace-htraced/src/go/format.sh deleted file mode 100755 index 46aa5b1..0000000 --- a/htrace-htraced/src/go/format.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# -# Reformats the HTrace code. -# -# ./format.sh Reformats all code. -# - -die() { - echo $@ - exit 1 -} - -# Check for gofmt. It should be installed whenever the go developement tools -# are installed. -which gofmt &> /dev/null -[ $? -ne 0 ] && die "You must install the gofmt code reformatting formatting tool." - -# Find go sources. We assume no newlines or whitespace in file names. -SCRIPT_DIR="$(cd "$( dirname $0 )" && pwd)" -find "${SCRIPT_DIR}/src" -noleaf -xdev -name '*.go' | xargs -L 1 gofmt -w diff --git a/htrace-htraced/src/go/gobuild.sh b/htrace-htraced/src/go/gobuild.sh deleted file mode 100755 index 81c9f7d..0000000 --- a/htrace-htraced/src/go/gobuild.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# -# Builds the HTrace server code. -# -# ./build.sh Builds the code. -# ./build.sh test Builds and runs all unit tests. -# ./build.sh bench Builds and runs all benchmarks -# - -die() { - echo $@ - exit 1 -} - -ACTION=install -if [ $# -gt 0 ]; then - ACTION="${1}" - shift -fi -RELEASE_VERSION=${RELEASE_VERSION:-unknown} - -# Set up directories. The build/ directory is where build dependencies and -# build binaries should go. -SCRIPT_DIR="$(cd "$( dirname $0 )" && pwd)" -export GOBIN="${SCRIPT_DIR}/build" -mkdir -p "${GOBIN}" || die "failed to mkdir -p ${GOBIN}" -cd "${GOBIN}" || die "failed to cd to ${SCRIPT_DIR}" -export GOPATH="${GOBIN}:${SCRIPT_DIR}" - -# Use the unsafe package when possible to get greater speed. For example, -# go-codec can bypass the overhead of converting between []byte and string in -# some cases when using unsafe. -TAGS="-tags unsafe" - -# Check for go -which go &> /dev/null -if [ $? -ne 0 ]; then - cat < /dev/null - [ $? -eq 0 ] && ldconfig=ldconfig -fi -if [ -n "${ldconfig}" ]; then - if "${ldconfig}" -p | grep -q libleveldb; then - : - else - echo "You must install the leveldb-devel package (or distro-specific equivalent.)" - exit 1 - fi -fi - -case $ACTION in -clean) - rm -rf -- "${GOBIN}" ${SCRIPT_DIR}/pkg - ;; -install) - # Ensure that we have the godep program. - PATH="${PATH}:${GOBIN}" - which godep &> /dev/null - if [ $? -ne 0 ]; then - echo "Installing godep..." - go get github.com/tools/godep || die "failed to get godep" - fi - - # Download dependencies into the build directory. - echo "godep restore..." - godep restore || die "failed to set up dependencies" - - # Discover the git version - GIT_VERSION=$(git rev-parse HEAD) - [ $? -eq 0 ] || GIT_VERSION="unknown" - - # Inject the release and git version into the htraced ldflags. - FLAGS="-X main.RELEASE_VERSION ${RELEASE_VERSION} -X main.GIT_VERSION ${GIT_VERSION}" - go install ${TAGS} -ldflags "${FLAGS}" -v org/apache/htrace/... "$@" - ;; -bench) - go test org/apache/htrace/... ${TAGS} -test.bench=. "$@" - ;; -*) - go ${ACTION} org/apache/htrace/... ${TAGS} "$@" - ;; -esac diff --git a/htrace-htraced/src/go/src/org/apache/htrace/client/client.go b/htrace-htraced/src/go/src/org/apache/htrace/client/client.go deleted file mode 100644 index 6a62e81..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/client/client.go +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package client - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "org/apache/htrace/common" - "org/apache/htrace/conf" -) - -// A golang client for htraced. -// TODO: fancier APIs for streaming spans in the background, optimize TCP stuff - -func NewClient(cnf *conf.Config) (*Client, error) { - hcl := Client{} - hcl.restAddr = cnf.Get(conf.HTRACE_WEB_ADDRESS) - hcl.hrpcAddr = cnf.Get(conf.HTRACE_HRPC_ADDRESS) - return &hcl, nil -} - -type Client struct { - // REST address of the htraced server. - restAddr string - - // HRPC address of the htraced server. - hrpcAddr string - - // The HRPC client, or null if it is not enabled. - hcr *hClient -} - -// Get the htraced server information. -func (hcl *Client) GetServerInfo() (*common.ServerInfo, error) { - buf, _, err := hcl.makeGetRequest("server/info") - if err != nil { - return nil, err - } - var info common.ServerInfo - err = json.Unmarshal(buf, &info) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ - "body %s: %s", string(buf), err.Error())) - } - return &info, nil -} - -// Get information about a trace span. Returns nil, nil if the span was not found. -func (hcl *Client) FindSpan(sid common.SpanId) (*common.Span, error) { - buf, rc, err := hcl.makeGetRequest(fmt.Sprintf("span/%016x", uint64(sid))) - if err != nil { - if rc == http.StatusNoContent { - return nil, nil - } - return nil, err - } - var span common.Span - err = json.Unmarshal(buf, &span) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error unmarshalling response "+ - "body %s: %s", string(buf), err.Error())) - } - return &span, nil -} - -func (hcl *Client) WriteSpans(req *common.WriteSpansReq) error { - if hcl.hrpcAddr == "" { - return hcl.writeSpansHttp(req) - } - hcr, err := newHClient(hcl.hrpcAddr) - if err != nil { - return err - } - defer hcr.Close() - return hcr.writeSpans(req) -} - -func (hcl *Client) writeSpansHttp(req *common.WriteSpansReq) error { - var w bytes.Buffer - var err error - for i := range req.Spans { - var buf []byte - buf, err = json.Marshal(req.Spans[i]) - if err != nil { - return errors.New(fmt.Sprintf("Error serializing span: %s", - err.Error())) - } - _, err = w.Write(buf) - if err != nil { - return errors.New(fmt.Sprintf("Error writing span: %s", - err.Error())) - } - _, err = w.Write([]byte{'\n'}) - //err = io.WriteString(&w, "\n") - if err != nil { - return errors.New(fmt.Sprintf("Error writing: %s", - err.Error())) - } - } - customHeaders := make(map[string]string) - if req.DefaultPid != "" { - customHeaders["htrace-pid"] = req.DefaultPid - } - _, _, err = hcl.makeRestRequest("POST", "writeSpans", - &w, customHeaders) - if err != nil { - return err - } - return nil -} - -// Find the child IDs of a given span ID. -func (hcl *Client) FindChildren(sid common.SpanId, lim int) ([]common.SpanId, error) { - buf, _, err := hcl.makeGetRequest(fmt.Sprintf("span/%016x/children?lim=%d", - uint64(sid), lim)) - if err != nil { - return nil, err - } - var spanIds []common.SpanId - err = json.Unmarshal(buf, &spanIds) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error: error unmarshalling response "+ - "body %s: %s", string(buf), err.Error())) - } - return spanIds, nil -} - -// Make a query -func (hcl *Client) Query(query *common.Query) ([]common.Span, error) { - in, err := json.Marshal(query) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error marshalling query: %s", err.Error())) - } - var out []byte - var url = fmt.Sprintf("query?query=%s", in) - out, _, err = hcl.makeGetRequest(url) - if err != nil { - return nil, err - } - var spans []common.Span - err = json.Unmarshal(out, &spans) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error unmarshalling results: %s", err.Error())) - } - return spans, nil -} - -var EMPTY = make(map[string]string) - -func (hcl *Client) makeGetRequest(reqName string) ([]byte, int, error) { - return hcl.makeRestRequest("GET", reqName, nil, EMPTY) -} - -// Make a general JSON REST request. -// Returns the request body, the response code, and the error. -// Note: if the response code is non-zero, the error will also be non-zero. -func (hcl *Client) makeRestRequest(reqType string, reqName string, reqBody io.Reader, - customHeaders map[string]string) ([]byte, int, error) { - url := fmt.Sprintf("http://%s/%s", - hcl.restAddr, reqName) - req, err := http.NewRequest(reqType, url, reqBody) - req.Header.Set("Content-Type", "application/json") - for k, v := range customHeaders { - req.Header.Set(k, v) - } - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, -1, errors.New(fmt.Sprintf("Error: error making http request to %s: %s\n", url, - err.Error())) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, resp.StatusCode, - errors.New(fmt.Sprintf("Error: got bad response status from %s: %s\n", url, resp.Status)) - } - var body []byte - body, err = ioutil.ReadAll(resp.Body) - if err != nil { - return nil, -1, errors.New(fmt.Sprintf("Error: error reading response body: %s\n", err.Error())) - } - return body, 0, nil -} - -// Dump all spans from the htraced daemon. -func (hcl *Client) DumpAll(lim int, out chan *common.Span) error { - defer func() { - close(out) - }() - searchId := common.SpanId(0) - for { - q := common.Query{ - Lim: lim, - Predicates: []common.Predicate{ - common.Predicate{ - Op: "ge", - Field: "spanid", - Val: searchId.String(), - }, - }, - } - spans, err := hcl.Query(&q) - if err != nil { - return errors.New(fmt.Sprintf("Error querying spans with IDs at or after "+ - "%s: %s", searchId.String(), err.Error())) - } - if len(spans) == 0 { - return nil - } - for i := range spans { - out <- &spans[i] - } - searchId = spans[len(spans)-1].Id + 1 - } -} - -func (hcl *Client) Close() { - if hcl.hcr != nil { - hcl.hcr.Close() - } - hcl.restAddr = "" - hcl.hcr = nil -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/client/hclient.go b/htrace-htraced/src/go/src/org/apache/htrace/client/hclient.go deleted file mode 100644 index 608dd59..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/client/hclient.go +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package client - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "github.com/ugorji/go/codec" - "io" - "net" - "net/rpc" - "org/apache/htrace/common" -) - -type hClient struct { - rpcClient *rpc.Client -} - -type HrpcClientCodec struct { - rwc io.ReadWriteCloser - length uint32 -} - -func (cdc *HrpcClientCodec) WriteRequest(req *rpc.Request, msg interface{}) error { - methodId := common.HrpcMethodNameToId(req.ServiceMethod) - if methodId == common.METHOD_ID_NONE { - return errors.New(fmt.Sprintf("HrpcClientCodec: Unknown method name %s", - req.ServiceMethod)) - } - mh := new(codec.MsgpackHandle) - mh.WriteExt = true - w := bytes.NewBuffer(make([]byte, 0, 2048)) - enc := codec.NewEncoder(w, mh) - err := enc.Encode(msg) - if err != nil { - return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+ - "message as msgpack: %s", err.Error())) - } - buf := w.Bytes() - if len(buf) > common.MAX_HRPC_BODY_LENGTH { - return errors.New(fmt.Sprintf("HrpcClientCodec: message body is %d "+ - "bytes, but the maximum message size is %d bytes.", - len(buf), common.MAX_HRPC_BODY_LENGTH)) - } - hdr := common.HrpcRequestHeader{ - Magic: common.HRPC_MAGIC, - MethodId: methodId, - Seq: req.Seq, - Length: uint32(len(buf)), - } - err = binary.Write(cdc.rwc, binary.LittleEndian, &hdr) - if err != nil { - return errors.New(fmt.Sprintf("Error writing header bytes: %s", - err.Error())) - } - _, err = cdc.rwc.Write(buf) - if err != nil { - return errors.New(fmt.Sprintf("Error writing body bytes: %s", - err.Error())) - } - return nil -} - -func (cdc *HrpcClientCodec) ReadResponseHeader(resp *rpc.Response) error { - hdr := common.HrpcResponseHeader{} - err := binary.Read(cdc.rwc, binary.LittleEndian, &hdr) - if err != nil { - return errors.New(fmt.Sprintf("Error reading response header "+ - "bytes: %s", err.Error())) - } - resp.ServiceMethod = common.HrpcMethodIdToMethodName(hdr.MethodId) - if resp.ServiceMethod == "" { - return errors.New(fmt.Sprintf("Error reading response header: "+ - "invalid method ID %d.", hdr.MethodId)) - } - resp.Seq = hdr.Seq - if hdr.ErrLength > 0 { - if hdr.ErrLength > common.MAX_HRPC_ERROR_LENGTH { - return errors.New(fmt.Sprintf("Error reading response header: "+ - "error message was %d bytes long, but "+ - "MAX_HRPC_ERROR_LENGTH is %d.", - hdr.ErrLength, common.MAX_HRPC_ERROR_LENGTH)) - } - buf := make([]byte, hdr.ErrLength) - var nread int - nread, err = cdc.rwc.Read(buf) - if uint32(nread) != hdr.ErrLength { - return errors.New(fmt.Sprintf("Error reading response header: "+ - "failed to read %d bytes of error message.", nread)) - } - if err != nil { - return errors.New(fmt.Sprintf("Error reading response header: "+ - "failed to read %d bytes of error message: %s", - nread, err.Error())) - } - resp.Error = string(buf) - } else { - resp.Error = "" - } - cdc.length = hdr.Length - return nil -} - -func (cdc *HrpcClientCodec) ReadResponseBody(body interface{}) error { - mh := new(codec.MsgpackHandle) - mh.WriteExt = true - dec := codec.NewDecoder(io.LimitReader(cdc.rwc, int64(cdc.length)), mh) - err := dec.Decode(body) - if err != nil { - return errors.New(fmt.Sprintf("Failed to read response body: %s", - err.Error())) - } - return nil -} - -func (cdc *HrpcClientCodec) Close() error { - return cdc.rwc.Close() -} - -func newHClient(hrpcAddr string) (*hClient, error) { - hcr := hClient{} - conn, err := net.Dial("tcp", hrpcAddr) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error contacting the HRPC server "+ - "at %s: %s", hrpcAddr, err.Error())) - } - hcr.rpcClient = rpc.NewClientWithCodec(&HrpcClientCodec{rwc: conn}) - return &hcr, nil -} - -func (hcr *hClient) writeSpans(req *common.WriteSpansReq) error { - resp := common.WriteSpansResp{} - return hcr.rpcClient.Call(common.METHOD_NAME_WRITE_SPANS, req, &resp) -} - -func (hcr *hClient) Close() { - hcr.rpcClient.Close() -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/log.go b/htrace-htraced/src/go/src/org/apache/htrace/common/log.go deleted file mode 100644 index 305ecf3..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/log.go +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "errors" - "fmt" - "org/apache/htrace/conf" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" -) - -// A logSink is a place logs can be written to. -type logSink struct { - path logPath - file *os.File - lock sync.Mutex - refCount int // protected by logFilesLock -} - -// Write to the logSink. -func (sink *logSink) write(str string) { - sink.lock.Lock() - defer sink.lock.Unlock() - _, err := sink.file.Write([]byte(str)) - if err != nil { - fmt.Fprintf(os.Stderr, "Error logging to '%s': %s\n", sink.path, err.Error()) - } -} - -// Unreference the logSink. If there are no more references, and the logSink is -// closeable, then we will close it here. -func (sink *logSink) Unref() { - logFilesLock.Lock() - defer logFilesLock.Unlock() - sink.refCount-- - if sink.refCount <= 0 { - if sink.path.IsCloseable() { - err := sink.file.Close() - if err != nil { - fmt.Fprintf(os.Stderr, "Error closing log file %s: %s\n", - sink.path, err.Error()) - } - } - logSinks[sink.path] = nil - } -} - -type logPath string - -// An empty LogPath represents "stdout." -const STDOUT_LOG_PATH = "" - -// Convert a path to a logPath. -func logPathFromString(path string) logPath { - if path == STDOUT_LOG_PATH { - return logPath("") - } - absPath, err := filepath.Abs(path) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to get absolute path of %s: %s\n", - path, err.Error()) - return logPath(path) - } - return logPath(absPath) -} - -// Convert the path to a human-readable string. -func (path logPath) String() string { - if path == "" { - return "(stdout)" - } else { - return string(path) - } -} - -// Return true if the path is closeable. stdout is not closeable. -func (path logPath) IsCloseable() bool { - return path != STDOUT_LOG_PATH -} - -func (path logPath) Open() *logSink { - if path == STDOUT_LOG_PATH { - return &logSink{path: path, file: os.Stdout} - } - file, err := os.OpenFile(string(path), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - sink := &logSink{path: STDOUT_LOG_PATH, file: os.Stdout} - fmt.Fprintf(os.Stderr, "Failed to open log file %s: %s\n", - path, err.Error()) - return sink - } - return &logSink{path: path, file: file} -} - -var logFilesLock sync.Mutex - -var logSinks map[logPath]*logSink = make(map[logPath]*logSink) - -func getOrCreateLogSink(pathStr string) *logSink { - path := logPathFromString(pathStr) - logFilesLock.Lock() - defer logFilesLock.Unlock() - sink := logSinks[path] - if sink == nil { - sink = path.Open() - logSinks[path] = sink - } - sink.refCount++ - return sink -} - -type Level int - -const ( - TRACE Level = iota - DEBUG - INFO - WARN - ERROR -) - -var levelToString map[Level]string = map[Level]string{ - TRACE: "TRACE", - DEBUG: "DEBUG", - INFO: "INFO", - WARN: "WARN", - ERROR: "ERROR", -} - -func (level Level) String() string { - return levelToString[level] -} - -func (level Level) LogString() string { - return level.String()[0:1] -} - -func LevelFromString(str string) (Level, error) { - for k, v := range levelToString { - if strings.ToLower(v) == strings.ToLower(str) { - return k, nil - } - } - var levelNames sort.StringSlice - levelNames = make([]string, len(levelToString)) - var i int - for _, v := range levelToString { - levelNames[i] = v - i++ - } - sort.Sort(levelNames) - return TRACE, errors.New(fmt.Sprintf("No such level as '%s'. Valid "+ - "levels are '%v'\n", str, levelNames)) -} - -type Logger struct { - sink *logSink - Level Level -} - -func NewLogger(faculty string, cnf *conf.Config) *Logger { - path, level := parseConf(faculty, cnf) - sink := getOrCreateLogSink(path) - return &Logger{sink: sink, Level: level} -} - -func parseConf(faculty string, cnf *conf.Config) (string, Level) { - facultyLogPathKey := faculty + "." + conf.HTRACE_LOG_PATH - var facultyLogPath string - if cnf.Contains(facultyLogPathKey) { - facultyLogPath = cnf.Get(facultyLogPathKey) - } else { - facultyLogPath = cnf.Get(conf.HTRACE_LOG_PATH) - } - facultyLogLevelKey := faculty + conf.HTRACE_LOG_LEVEL - var facultyLogLevelStr string - if cnf.Contains(facultyLogLevelKey) { - facultyLogLevelStr = cnf.Get(facultyLogLevelKey) - } else { - facultyLogLevelStr = cnf.Get(conf.HTRACE_LOG_LEVEL) - } - level, err := LevelFromString(facultyLogLevelStr) - if err != nil { - fmt.Fprintf(os.Stderr, "Error configuring log level: %s. Using TRACE.\n") - level = TRACE - } - return facultyLogPath, level -} - -func (lg *Logger) Trace(str string) { - lg.Write(TRACE, str) -} - -func (lg *Logger) Tracef(format string, v ...interface{}) { - lg.Write(TRACE, fmt.Sprintf(format, v...)) -} - -func (lg *Logger) Debug(str string) { - lg.Write(DEBUG, str) -} - -func (lg *Logger) Debugf(format string, v ...interface{}) { - lg.Write(DEBUG, fmt.Sprintf(format, v...)) -} - -func (lg *Logger) Info(str string) { - lg.Write(INFO, str) -} - -func (lg *Logger) Infof(format string, v ...interface{}) { - lg.Write(INFO, fmt.Sprintf(format, v...)) -} - -func (lg *Logger) Warn(str string) error { - lg.Write(WARN, str) - return errors.New(str) -} - -func (lg *Logger) Warnf(format string, v ...interface{}) error { - str := fmt.Sprintf(format, v...) - lg.Write(WARN, str) - return errors.New(str) -} - -func (lg *Logger) Error(str string) error { - lg.Write(ERROR, str) - return errors.New(str) -} - -func (lg *Logger) Errorf(format string, v ...interface{}) error { - str := fmt.Sprintf(format, v...) - lg.Write(ERROR, str) - return errors.New(str) -} - -func (lg *Logger) Write(level Level, str string) { - if level >= lg.Level { - lg.sink.write(time.Now().Format(time.RFC3339) + " " + - level.LogString() + ": " + str) - } -} - -// -// A few functions which can be used to determine if a certain level of tracing -// is enabled. These are useful in situations when evaluating the parameters -// of a logging function is expensive. (Note, however, that we don't pay the -// cost of string concatenation and manipulation when a log message doesn't -// trigger.) -// - -func (lg *Logger) TraceEnabled() bool { - return lg.Level >= TRACE -} - -func (lg *Logger) DebugEnabled() bool { - return lg.Level >= DEBUG -} - -func (lg *Logger) InfoEnabled() bool { - return lg.Level >= INFO -} - -func (lg *Logger) WarnEnabled() bool { - return lg.Level >= WARN -} - -func (lg *Logger) ErrorEnabled() bool { - return lg.Level >= ERROR -} - -func (lg *Logger) Close() { - lg.sink.Unref() - lg.sink = nil -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/log_test.go b/htrace-htraced/src/go/src/org/apache/htrace/common/log_test.go deleted file mode 100644 index b415ce2..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/log_test.go +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "bufio" - "fmt" - "io" - "io/ioutil" - "org/apache/htrace/conf" - "os" - "strings" - "testing" -) - -func newLogger(faculty string, args ...string) *Logger { - cnfBld := conf.Builder{Defaults: conf.DEFAULTS} - cnf, err := cnfBld.Build() - if err != nil { - panic(fmt.Sprintf("failed to create conf: %s", err.Error())) - } - cnf2 := cnf.Clone(args...) - lg := NewLogger(faculty, cnf2) - return lg -} - -func TestNewLogger(t *testing.T) { - lg := newLogger("foo", "log.level", "TRACE") - lg.Close() -} - -func verifyLines(t *testing.T, rdr io.Reader, lines []string) { - scanner := bufio.NewScanner(rdr) - lineIdx := 0 - for scanner.Scan() { - line := scanner.Text() - if !strings.Contains(line, lines[lineIdx]) { - t.Fatalf("Error on line %d: didn't find substring '%s' in line '%s'\n", - (lineIdx + 1), lines[lineIdx], line) - } - lineIdx++ - } - if err := scanner.Err(); err != nil { - t.Fatal(err.Error()) - } -} - -func TestFileLogs(t *testing.T) { - tempDir, err := ioutil.TempDir(os.TempDir(), "testFileLogs") - if err != nil { - panic(fmt.Sprintf("error creating tempdir: %s\n", err.Error())) - } - defer os.RemoveAll(tempDir) - logPath := tempDir + conf.PATH_SEP + "log" - lg := newLogger("foo", "log.level", "DEBUG", - "foo.log.level", "INFO", - "log.path", logPath) - lg.Tracef("Non-important stuff, ignore this.\n") - lg.Infof("problem with the foobar\n") - lg.Tracef("More non-important stuff, also ignore this.\n") - lg.Infof("and another problem with the foobar\n") - logFile, err := os.Open(logPath) - if err != nil { - t.Fatalf("failed to open file %s: %s\n", logPath, err.Error()) - } - verifyLines(t, logFile, []string{ - "problem with the foobar", - "and another problem with the foobar", - }) - logFile.Close() - lg.Close() -} - -func TestMultipleFileLogs(t *testing.T) { - tempDir, err := ioutil.TempDir(os.TempDir(), "testMultipleFileLogs") - if err != nil { - panic(fmt.Sprintf("error creating tempdir: %s\n", err.Error())) - } - defer os.RemoveAll(tempDir) - logPath := tempDir + conf.PATH_SEP + "log" - fooLg := newLogger("foo", "log.level", "DEBUG", - "foo.log.level", "INFO", - "log.path", logPath) - fooLg.Infof("The foo needs maintenance.\n") - barLg := newLogger("bar", "log.level", "DEBUG", - "foo.log.level", "INFO", - "log.path", logPath) - barLg.Debugf("The bar is open\n") - fooLg.Errorf("Fizz buzz\n") - logFile, err := os.Open(logPath) - if err != nil { - t.Fatalf("failed to open file %s: %s\n", logPath, err.Error()) - } - fooLg.Tracef("Fizz buzz2\n") - barLg.Tracef("Fizz buzz3\n") - verifyLines(t, logFile, []string{ - "The foo needs maintenance.", - "The bar is open", - "Fizz buzz", - "Fizz buzz3", - }) - logFile.Close() - fooLg.Close() - barLg.Close() -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/process.go b/htrace-htraced/src/go/src/org/apache/htrace/common/process.go deleted file mode 100644 index d138178..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/process.go +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "bufio" - "org/apache/htrace/conf" - "os" - "os/signal" - "syscall" -) - -func LoadApplicationConfig() *conf.Config { - cnf, dlog := conf.LoadApplicationConfig() - lg := NewLogger("conf", cnf) - defer lg.Close() - if lg.Level <= DEBUG { - // Print out the debug information from loading the configuration. - scanner := bufio.NewScanner(dlog) - for scanner.Scan() { - lg.Debugf(scanner.Text() + "\n") - } - } - return cnf -} - -func InstallSignalHandlers(cnf *conf.Config) { - fatalSigs := []os.Signal{ - os.Interrupt, - os.Kill, - syscall.SIGINT, - syscall.SIGABRT, - syscall.SIGALRM, - syscall.SIGBUS, - syscall.SIGFPE, - syscall.SIGILL, - syscall.SIGQUIT, - syscall.SIGSEGV, - syscall.SIGTERM, - } - sigChan := make(chan os.Signal, len(fatalSigs)) - signal.Notify(sigChan, fatalSigs...) - lg := NewLogger("exit", cnf) - go func() { - sig := <-sigChan - lg.Errorf("Terminating on signal: %v\n", sig) - lg.Close() - os.Exit(1) - }() -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/query.go b/htrace-htraced/src/go/src/org/apache/htrace/common/query.go deleted file mode 100644 index 8c9128f..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/query.go +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "encoding/json" -) - -// -// Represents queries that can be sent to htraced. -// -// Each query consists of set of predicates that will be 'AND'ed together to -// return a set of spans. Predicates contain an operation, a field, and a -// value. -// -// For example, a query might be "return the first 100 spans between 5:00pm -// and 5:01pm" This query would have two predicates: time greater than or -// equal to 5:00pm, and time less than or equal to 5:01pm. -// In HTrace, times are always expressed in milliseconds since the Epoch. -// So this would become: -// { "lim" : 100, "pred" : [ -// { "op" : "ge", "field" : "begin", "val" : 1234 }, -// { "op" : "le", "field" : "begin", "val" : 5678 }, -// ] } -// -// Where '1234' and '5678' were replaced by times since the epoch in -// milliseconds. -// - -type Op string - -const ( - CONTAINS Op = "cn" - EQUALS Op = "eq" - LESS_THAN_OR_EQUALS Op = "le" - GREATER_THAN_OR_EQUALS Op = "ge" - GREATER_THAN Op = "gt" -) - -func (op Op) IsDescending() bool { - return op == LESS_THAN_OR_EQUALS -} - -func (op Op) IsValid() bool { - ops := ValidOps() - for i := range ops { - if ops[i] == op { - return true - } - } - return false -} - -func ValidOps() []Op { - return []Op{CONTAINS, EQUALS, LESS_THAN_OR_EQUALS, GREATER_THAN_OR_EQUALS, - GREATER_THAN} -} - -type Field string - -const ( - SPAN_ID Field = "spanid" - DESCRIPTION Field = "description" - BEGIN_TIME Field = "begin" - END_TIME Field = "end" - DURATION Field = "duration" -) - -func (field Field) IsValid() bool { - fields := ValidFields() - for i := range fields { - if fields[i] == field { - return true - } - } - return false -} - -func ValidFields() []Field { - return []Field{SPAN_ID, DESCRIPTION, BEGIN_TIME, END_TIME, DURATION} -} - -type Predicate struct { - Op Op `json:"op"` - Field Field `json:"field"` - Val string `val:"val"` -} - -func (pred *Predicate) String() string { - buf, err := json.Marshal(pred) - if err != nil { - panic(err) - } - return string(buf) -} - -type Query struct { - Predicates []Predicate `json:"pred"` - Lim int `json:"lim"` - Prev *Span `json:"prev"` -} - -func (query *Query) String() string { - buf, err := json.Marshal(query) - if err != nil { - panic(err) - } - return string(buf) -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/query_test.go b/htrace-htraced/src/go/src/org/apache/htrace/common/query_test.go deleted file mode 100644 index 2697d9c..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/query_test.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "testing" -) - -func TestValidOps(t *testing.T) { - for i := range ValidOps() { - op := ValidOps()[i] - if !op.IsValid() { - t.Fatalf("op %s was in ValidOps, but IsValid returned false.\n", op) - } - } - invalidOp := Op("completelybogus") - if invalidOp.IsValid() { - t.Fatalf("op %s was invalid, but IsValid returned true.\n", invalidOp) - } -} - -func TestValidFields(t *testing.T) { - for i := range ValidFields() { - field := ValidFields()[i] - if !field.IsValid() { - t.Fatalf("field %s was in ValidFields, but IsValid returned false.\n", field) - } - } - invalidField := Field("completelybogus") - if invalidField.IsValid() { - t.Fatalf("field %s was invalid, but IsValid returned true.\n", invalidField) - } -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/rest.go b/htrace-htraced/src/go/src/org/apache/htrace/common/rest.go deleted file mode 100644 index b898ca4..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/rest.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -// Info returned by /server/info -type ServerInfo struct { - // The server release version. - ReleaseVersion string - - // The git hash that this software was built with. - GitVersion string -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/rpc.go b/htrace-htraced/src/go/src/org/apache/htrace/common/rpc.go deleted file mode 100644 index fe50a44..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/rpc.go +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -// The 4-byte magic number which is sent first in the HRPC header -const HRPC_MAGIC = 0x43525448 - -// Method ID codes. Do not reorder these. -const ( - METHOD_ID_NONE = 0 - METHOD_ID_WRITE_SPANS = iota -) - -const METHOD_NAME_WRITE_SPANS = "HrpcHandler.WriteSpans" - -// Maximum length of the error message passed in an HRPC response -const MAX_HRPC_ERROR_LENGTH = 4 * 1024 * 1024 - -// Maximum length of HRPC message body -const MAX_HRPC_BODY_LENGTH = 64 * 1024 * 1024 - -// A request to write spans to htraced. -type WriteSpansReq struct { - DefaultPid string - Spans []*Span -} - -// A response to a WriteSpansReq -type WriteSpansResp struct { -} - -// The header which is sent over the wire for HRPC -type HrpcRequestHeader struct { - Magic uint32 - MethodId uint32 - Seq uint64 - Length uint32 -} - -// The response which is sent over the wire for HRPC -type HrpcResponseHeader struct { - Seq uint64 - MethodId uint32 - ErrLength uint32 - Length uint32 -} - -func HrpcMethodIdToMethodName(id uint32) string { - switch id { - case METHOD_ID_WRITE_SPANS: - return METHOD_NAME_WRITE_SPANS - default: - return "" - } -} - -func HrpcMethodNameToId(name string) uint32 { - switch name { - case METHOD_NAME_WRITE_SPANS: - return METHOD_ID_WRITE_SPANS - default: - return METHOD_ID_NONE - } -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/span.go b/htrace-htraced/src/go/src/org/apache/htrace/common/span.go deleted file mode 100644 index b276844..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/span.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "encoding/json" - "errors" - "fmt" - "strconv" -) - -// -// Represents a trace span. -// -// Compatibility notes: -// When converting to JSON, we store the 64-bit numbers as hexadecimal strings rather than as -// integers. This is because JavaScript lacks the ability to handle 64-bit integers. Numbers above -// about 55 bits will be rounded by Javascript. Since the Javascript UI is a primary consumer of -// this JSON data, we have to simply pass it as a string. -// - -type TraceInfoMap map[string]string - -type TimelineAnnotation struct { - Time int64 `json:"t"` - Msg string `json:"m"` -} - -type SpanId uint64 - -func (id SpanId) String() string { - return fmt.Sprintf("%016x", uint64(id)) -} - -func (id SpanId) Val() uint64 { - return uint64(id) -} - -func (id SpanId) MarshalJSON() ([]byte, error) { - return []byte(`"` + fmt.Sprintf("%016x", uint64(id)) + `"`), nil -} - -type SpanSlice []*Span - -func (s SpanSlice) Len() int { - return len(s) -} - -func (s SpanSlice) Less(i, j int) bool { - return s[i].Id < s[j].Id -} - -func (s SpanSlice) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -type SpanIdSlice []SpanId - -func (s SpanIdSlice) Len() int { - return len(s) -} - -func (s SpanIdSlice) Less(i, j int) bool { - return s[i] < s[j] -} - -func (s SpanIdSlice) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -const DOUBLE_QUOTE = 0x22 - -func (id *SpanId) UnmarshalJSON(b []byte) error { - if b[0] != DOUBLE_QUOTE { - return errors.New("Expected spanID to start with a string quote.") - } - if b[len(b)-1] != DOUBLE_QUOTE { - return errors.New("Expected spanID to end with a string quote.") - } - return id.FromString(string(b[1 : len(b)-1])) -} - -func (id *SpanId) FromString(str string) error { - v, err := strconv.ParseUint(str, 16, 64) - if err != nil { - return err - } - *id = SpanId(v) - return nil -} - -type SpanData struct { - Begin int64 `json:"b"` - End int64 `json:"e"` - Description string `json:"d"` - TraceId SpanId `json:"i"` - Parents []SpanId `json:"p"` - Info TraceInfoMap `json:"n,omitempty"` - ProcessId string `json:"r"` - TimelineAnnotations []TimelineAnnotation `json:"t,omitempty"` -} - -type Span struct { - Id SpanId `json:"s"` - SpanData -} - -func (span *Span) ToJson() []byte { - jbytes, err := json.Marshal(*span) - if err != nil { - panic(err) - } - return jbytes -} - -func (span *Span) String() string { - return string(span.ToJson()) -} - -// Compute the span duration. We ignore overflow since we never deal with negative times. -func (span *Span) Duration() int64 { - return span.End - span.Begin -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/span_test.go b/htrace-htraced/src/go/src/org/apache/htrace/common/span_test.go deleted file mode 100644 index f218b3a..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/span_test.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "testing" -) - -func TestSpanToJson(t *testing.T) { - t.Parallel() - span := Span{Id: 2305843009213693952, - SpanData: SpanData{ - Begin: 123, - End: 456, - Description: "getFileDescriptors", - TraceId: 999, - Parents: []SpanId{}, - ProcessId: "testProcessId", - }} - ExpectStrEqual(t, - `{"s":"2000000000000000","b":123,"e":456,"d":"getFileDescriptors","i":"00000000000003e7","p":[],"r":"testProcessId"}`, - string(span.ToJson())) -} - -func TestAnnotatedSpanToJson(t *testing.T) { - t.Parallel() - span := Span{Id: 1305813009213693952, - SpanData: SpanData{ - Begin: 1234, - End: 4567, - Description: "getFileDescriptors2", - TraceId: 999, - Parents: []SpanId{}, - ProcessId: "testAnnotatedProcessId", - TimelineAnnotations: []TimelineAnnotation{ - TimelineAnnotation{ - Time: 7777, - Msg: "contactedServer", - }, - TimelineAnnotation{ - Time: 8888, - Msg: "passedFd", - }, - }, - }} - ExpectStrEqual(t, - `{"s":"121f2e036d442000","b":1234,"e":4567,"d":"getFileDescriptors2","i":"00000000000003e7","p":[],"r":"testAnnotatedProcessId","t":[{"t":7777,"m":"contactedServer"},{"t":8888,"m":"passedFd"}]}`, - string(span.ToJson())) -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/common/test_util.go b/htrace-htraced/src/go/src/org/apache/htrace/common/test_util.go deleted file mode 100644 index 871c847..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/common/test_util.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package common - -import ( - "fmt" - "testing" - "time" -) - -type Int64Slice []int64 - -func (p Int64Slice) Len() int { return len(p) } -func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } -func (p Int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -type SupplierFun func() bool - -// -// Wait for a configurable amount of time for a precondition to become true. -// -// Example: -// WaitFor(time.Minute * 1, time.Millisecond * 1, func() bool { -// return ht.Store.GetStatistics().NumSpansWritten >= 3 -// }) -// -func WaitFor(dur time.Duration, poll time.Duration, fun SupplierFun) { - if poll == 0 { - poll = dur / 10 - } - if poll <= 0 { - panic("Can't have a polling time less than zero.") - } - endTime := time.Now().Add(dur) - for { - if fun() { - return - } - if !time.Now().Before(endTime) { - break - } - time.Sleep(poll) - } - panic(fmt.Sprintf("Timed out after %s", dur)) -} - -// Trigger a test failure if two strings are not equal. -func ExpectStrEqual(t *testing.T, expect string, actual string) { - if expect != actual { - t.Fatalf("Expected:\n%s\nGot:\n%s\n", expect, actual) - } -} - -// Trigger a test failure if the JSON representation of two spans are not equals. -func ExpectSpansEqual(t *testing.T, spanA *Span, spanB *Span) { - ExpectStrEqual(t, string(spanA.ToJson()), string(spanB.ToJson())) -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/conf/config.go b/htrace-htraced/src/go/src/org/apache/htrace/conf/config.go deleted file mode 100644 index 6093649..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/conf/config.go +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package conf - -import ( - "bufio" - "bytes" - "fmt" - "io" - "log" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "syscall" -) - -// -// The configuration code for HTraced. -// -// HTraced can be configured via Hadoop-style XML configuration files, or by passing -Dkey=value -// command line arguments. Command-line arguments without an equals sign, such as "-Dkey", will be -// treated as setting the key to "true". -// -// Configuration key constants should be defined in config_keys.go. Each key should have a default, -// which will be used if the user supplies no value, or supplies an invalid value. -// For that reason, it is not necessary for the Get, GetInt, etc. functions to take a default value -// argument. -// -// Configuration objects are immutable. However, you can make a copy of a configuration which adds -// some changes using Configuration#Clone(). -// - -type Config struct { - settings map[string]string - defaults map[string]string -} - -type Builder struct { - // If non-nil, the XML configuration file to read. - Reader io.Reader - - // If non-nil, the configuration values to use. - Values map[string]string - - // If non-nil, the default configuration values to use. - Defaults map[string]string - - // If non-nil, the command-line arguments to use. - Argv []string -} - -func getHTracedConfDirs(dlog io.Writer) []string { - confDir := os.Getenv("HTRACED_CONF_DIR") - io.WriteString(dlog, fmt.Sprintf("HTRACED_CONF_DIR=%s\n", confDir)) - paths := filepath.SplitList(confDir) - if len(paths) < 1 { - return []string{"."} - } - return paths -} - -// Load a configuration from the application's argv, configuration file, and the standard -// defaults. -func LoadApplicationConfig() (*Config, io.Reader) { - dlog := new(bytes.Buffer) - reader := openFile(CONFIG_FILE_NAME, getHTracedConfDirs(dlog), dlog) - bld := Builder{} - if reader != nil { - defer reader.Close() - bld.Reader = bufio.NewReader(reader) - } - bld.Argv = os.Args[1:] - bld.Defaults = DEFAULTS - cnf, err := bld.Build() - if err != nil { - log.Fatal("Error building configuration: " + err.Error()) - } - os.Args = append(os.Args[0:1], bld.Argv...) - keys := make(sort.StringSlice, 0, 20) - for k, _ := range cnf.settings { - keys = append(keys, k) - } - sort.Sort(keys) - for i := range keys { - io.WriteString(dlog, fmt.Sprintf("%s = %s\n", - keys[i], cnf.settings[keys[i]])) - } - return cnf, dlog -} - -// Attempt to open a configuration file somewhere on the provided list of paths. -func openFile(cnfName string, paths []string, dlog io.Writer) io.ReadCloser { - for p := range paths { - path := fmt.Sprintf("%s%c%s", paths[p], os.PathSeparator, cnfName) - file, err := os.Open(path) - if err == nil { - io.WriteString(dlog, fmt.Sprintf("Reading configuration from %s.\n", path)) - return file - } - if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOENT { - continue - } - io.WriteString(dlog, fmt.Sprintf("Error opening %s for read: %s\n", path, err.Error())) - } - return nil -} - -// Try to parse a command-line element as a key=value pair. -func parseAsConfigFlag(flag string) (string, string) { - var confPart string - if strings.HasPrefix(flag, "-D") { - confPart = flag[2:] - } else if strings.HasPrefix(flag, "--D") { - confPart = flag[3:] - } else { - return "", "" - } - if len(confPart) == 0 { - return "", "" - } - idx := strings.Index(confPart, "=") - if idx == -1 { - return confPart, "true" - } - return confPart[0:idx], confPart[idx+1:] -} - -// Build a new configuration object from the provided conf.Builder. -func (bld *Builder) Build() (*Config, error) { - // Load values and defaults - cnf := Config{} - cnf.settings = make(map[string]string) - if bld.Values != nil { - for k, v := range bld.Values { - cnf.settings[k] = v - } - } - cnf.defaults = make(map[string]string) - if bld.Defaults != nil { - for k, v := range bld.Defaults { - cnf.defaults[k] = v - } - } - - // Process the configuration file, if we have one - if bld.Reader != nil { - parseXml(bld.Reader, cnf.settings) - } - - // Process command line arguments - var i int - for i < len(bld.Argv) { - str := bld.Argv[i] - key, val := parseAsConfigFlag(str) - if key != "" { - if val == "" { - cnf.settings[key] = "true" - } else { - cnf.settings[key] = val - } - bld.Argv = append(bld.Argv[:i], bld.Argv[i+1:]...) - } else { - i++ - } - } - return &cnf, nil -} - -// Returns true if the configuration has a non-default value for the given key. -func (cnf *Config) Contains(key string) bool { - _, ok := cnf.settings[key] - return ok -} - -// Get a string configuration key. -func (cnf *Config) Get(key string) string { - ret := cnf.settings[key] - if ret != "" { - return ret - } - return cnf.defaults[key] -} - -// Get a boolean configuration key. -func (cnf *Config) GetBool(key string) bool { - str := cnf.settings[key] - ret, err := strconv.ParseBool(str) - if err == nil { - return ret - } - str = cnf.defaults[key] - ret, err = strconv.ParseBool(str) - if err == nil { - return ret - } - return false -} - -// Get an integer configuration key. -func (cnf *Config) GetInt(key string) int { - str := cnf.settings[key] - ret, err := strconv.Atoi(str) - if err == nil { - return ret - } - str = cnf.defaults[key] - ret, err = strconv.Atoi(str) - if err == nil { - return ret - } - return 0 -} - -// Get an int64 configuration key. -func (cnf *Config) GetInt64(key string) int64 { - str := cnf.settings[key] - ret, err := strconv.ParseInt(str, 10, 64) - if err == nil { - return ret - } - str = cnf.defaults[key] - ret, err = strconv.ParseInt(str, 10, 64) - if err == nil { - return ret - } - return 0 -} - -// Make a deep copy of the given configuration. -// Optionally, you can specify particular key/value pairs to change. -// Example: -// cnf2 := cnf.Copy("my.changed.key", "my.new.value") -func (cnf *Config) Clone(args ...string) *Config { - if len(args)%2 != 0 { - panic("The arguments to Config#copy are key1, value1, " + - "key2, value2, and so on. You must specify an even number of arguments.") - } - ncnf := &Config{defaults: cnf.defaults} - ncnf.settings = make(map[string]string) - for k, v := range cnf.settings { - ncnf.settings[k] = v - } - for i := 0; i < len(args); i += 2 { - ncnf.settings[args[i]] = args[i+1] - } - return ncnf -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/conf/config_keys.go b/htrace-htraced/src/go/src/org/apache/htrace/conf/config_keys.go deleted file mode 100644 index ccb09e0..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/conf/config_keys.go +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package conf - -import ( - "fmt" - "os" -) - -// -// Configuration keys for HTrace. -// - -// The platform-specific path separator. Usually slash. -var PATH_SEP string = fmt.Sprintf("%c", os.PathSeparator) - -// The platform-specific path list separator. Usually colon. -var PATH_LIST_SEP string = fmt.Sprintf("%c", os.PathListSeparator) - -// The name of the XML configuration file to look for. -const CONFIG_FILE_NAME = "htraced-conf.xml" - -// An environment variable containing a list of paths to search for the -// configuration file in. -const HTRACED_CONF_DIR = "HTRACED_CONF_DIR" - -// The web address to start the REST server on. -const HTRACE_WEB_ADDRESS = "web.address" - -// The default port for the Htrace web address. -const HTRACE_WEB_ADDRESS_DEFAULT_PORT = 9095 - -// The web address to start the REST server on. -const HTRACE_HRPC_ADDRESS = "hrpc.address" - -// The default port for the Htrace HRPC address. -const HTRACE_HRPC_ADDRESS_DEFAULT_PORT = 9075 - -// The directories to put the data store into. Separated by PATH_LIST_SEP. -const HTRACE_DATA_STORE_DIRECTORIES = "data.store.directories" - -// Boolean key which indicates whether we should clear data on startup. -const HTRACE_DATA_STORE_CLEAR = "data.store.clear" - -// How many writes to buffer before applying backpressure to span senders. -const HTRACE_DATA_STORE_SPAN_BUFFER_SIZE = "data.store.span.buffer.size" - -// Path to put the logs from htrace, or the empty string to use stdout. -const HTRACE_LOG_PATH = "log.path" - -// The log level to use for the logs in htrace. -const HTRACE_LOG_LEVEL = "log.level" - -// A host:port pair to send information to on startup. This is used in unit -// tests to determine the (random) port of the htraced process that has been -// started. -const HTRACE_STARTUP_NOTIFICATION_ADDRESS = "startup.notification.address" - -// Default values for HTrace configuration keys. -var DEFAULTS = map[string]string{ - HTRACE_WEB_ADDRESS: fmt.Sprintf("0.0.0.0:%d", HTRACE_WEB_ADDRESS_DEFAULT_PORT), - HTRACE_HRPC_ADDRESS: fmt.Sprintf("0.0.0.0:%d", HTRACE_HRPC_ADDRESS_DEFAULT_PORT), - HTRACE_DATA_STORE_DIRECTORIES: PATH_SEP + "tmp" + PATH_SEP + "htrace1" + - PATH_LIST_SEP + PATH_SEP + "tmp" + PATH_SEP + "htrace2", - HTRACE_DATA_STORE_CLEAR: "false", - HTRACE_DATA_STORE_SPAN_BUFFER_SIZE: "100", - HTRACE_LOG_PATH: "", - HTRACE_LOG_LEVEL: "INFO", -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/conf/config_test.go b/htrace-htraced/src/go/src/org/apache/htrace/conf/config_test.go deleted file mode 100644 index 42c1c71..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/conf/config_test.go +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package conf - -import ( - "bytes" - "os" - "strings" - "testing" -) - -// Test that parsing command-line arguments of the form -Dfoo=bar works. -func TestParseArgV(t *testing.T) { - t.Parallel() - argv := []string{"-Dfoo=bar", "-Dbaz=123", "-DsillyMode"} - bld := &Builder{Argv: argv} - cnf, err := bld.Build() - if err != nil { - t.Fatal() - } - if "bar" != cnf.Get("foo") { - t.Fatal() - } - if 123 != cnf.GetInt("baz") { - t.Fatal() - } - if !cnf.GetBool("sillyMode") { - t.Fatal() - } - if cnf.GetBool("otherSillyMode") { - t.Fatal() - } -} - -// Test that default values work. -// Defaults are used only when the configuration option is not present or can't be parsed. -func TestDefaults(t *testing.T) { - t.Parallel() - argv := []string{"-Dfoo=bar", "-Dbaz=invalidNumber"} - defaults := map[string]string{ - "foo": "notbar", - "baz": "456", - "foo2": "4611686018427387904", - } - bld := &Builder{Argv: argv, Defaults: defaults} - cnf, err := bld.Build() - if err != nil { - t.Fatal() - } - if "bar" != cnf.Get("foo") { - t.Fatal() - } - if 456 != cnf.GetInt("baz") { - t.Fatal() - } - if 4611686018427387904 != cnf.GetInt64("foo2") { - t.Fatal() - } -} - -// Test that we can parse our XML configuration file. -func TestXmlConfigurationFile(t *testing.T) { - t.Parallel() - xml := ` - - - - - foo.bar - 123 - - - foo.baz - xmlValue - - - -` - xmlReader := strings.NewReader(xml) - argv := []string{"-Dfoo.bar=456"} - defaults := map[string]string{ - "foo.bar": "789", - "cmdline.opt": "4611686018427387904", - } - bld := &Builder{Argv: argv, Defaults: defaults, Reader: xmlReader} - cnf, err := bld.Build() - if err != nil { - t.Fatal() - } - // The command-line argument takes precedence over the XML and the defaults. - if 456 != cnf.GetInt("foo.bar") { - t.Fatal() - } - if "xmlValue" != cnf.Get("foo.baz") { - t.Fatalf("foo.baz = %s", cnf.Get("foo.baz")) - } - if "" != cnf.Get("commented.out") { - t.Fatal() - } - if 4611686018427387904 != cnf.GetInt64("cmdline.opt") { - t.Fatal() - } -} - -// Test our handling of the HTRACE_CONF_DIR environment variable. -func TestGetHTracedConfDirs(t *testing.T) { - os.Setenv("HTRACED_CONF_DIR", "") - dlog := new(bytes.Buffer) - dirs := getHTracedConfDirs(dlog) - if len(dirs) != 1 || dirs[0] != "." { - t.Fatal() - } - os.Setenv("HTRACED_CONF_DIR", "/foo/bar:/baz") - dirs = getHTracedConfDirs(dlog) - if len(dirs) != 2 || dirs[0] != "/foo/bar" || dirs[1] != "/baz" { - t.Fatal() - } -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/conf/xml.go b/htrace-htraced/src/go/src/org/apache/htrace/conf/xml.go deleted file mode 100644 index de14bc5..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/conf/xml.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package conf - -import ( - "encoding/xml" - "io" - "log" -) - -type configuration struct { - Properties []propertyXml `xml:"property"` -} - -type propertyXml struct { - Name string `xml:"name"` - Value string `xml:"value"` -} - -// Parse an XML configuration file. -func parseXml(reader io.Reader, m map[string]string) error { - dec := xml.NewDecoder(reader) - configurationXml := configuration{} - err := dec.Decode(&configurationXml) - if err != nil { - return err - } - props := configurationXml.Properties - for p := range props { - key := props[p].Name - value := props[p].Value - if key == "" { - log.Println("Warning: ignoring element with missing or empty .") - continue - } - if value == "" { - log.Println("Warning: ignoring element with key " + key + " with missing or empty .") - continue - } - //log.Printf("setting %s to %s\n", key, value) - m[key] = value - } - return nil -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htrace/cmd.go b/htrace-htraced/src/go/src/org/apache/htrace/htrace/cmd.go deleted file mode 100644 index 38cdb58..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htrace/cmd.go +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bufio" - "bytes" - "encoding/json" - "errors" - "fmt" - "github.com/alecthomas/kingpin" - "io" - htrace "org/apache/htrace/client" - "org/apache/htrace/common" - "org/apache/htrace/conf" - "os" - "time" -) - -var RELEASE_VERSION string -var GIT_VERSION string - -const EXIT_SUCCESS = 0 -const EXIT_FAILURE = 1 - -var verbose *bool - -const USAGE = `The Apache HTrace command-line tool. This tool retrieves and modifies settings and -other data on a running htraced daemon. - -If we find an ` + conf.CONFIG_FILE_NAME + ` configuration file in the list of directories -specified in ` + conf.HTRACED_CONF_DIR + `, we will use that configuration; otherwise, -the defaults will be used. -` - -func main() { - // Load htraced configuration - cnf := common.LoadApplicationConfig() - - // Parse argv - app := kingpin.New(os.Args[0], USAGE) - app.Flag("Dmy.key", "Set configuration key 'my.key' to 'my.value'. Replace 'my.key' "+ - "with any key you want to set.").Default("my.value").String() - addr := app.Flag("addr", "Server address.").String() - verbose = app.Flag("verbose", "Verbose.").Default("false").Bool() - version := app.Command("version", "Print the version of this program.") - serverInfo := app.Command("serverInfo", "Print information retrieved from an htraced server.") - findSpan := app.Command("findSpan", "Print information about a trace span with a given ID.") - findSpanId := findSpan.Arg("id", "Span ID to find. Example: 0x123456789abcdef").Required().Uint64() - findChildren := app.Command("findChildren", "Print out the span IDs that are children of a given span ID.") - parentSpanId := findChildren.Arg("id", "Span ID to print children for. Example: 0x123456789abcdef"). - Required().Uint64() - childLim := findChildren.Flag("lim", "Maximum number of child IDs to print.").Default("20").Int() - loadFile := app.Command("loadFile", "Write whitespace-separated JSON spans from a file to the server.") - loadFilePath := loadFile.Arg("path", - "A file containing whitespace-separated span JSON.").Required().String() - loadJson := app.Command("load", "Write JSON spans from the command-line to the server.") - loadJsonArg := loadJson.Arg("json", "A JSON span to write to the server.").Required().String() - dumpAll := app.Command("dumpAll", "Dump all spans from the htraced daemon.") - dumpAllOutPath := dumpAll.Arg("path", "The path to dump the trace spans to.").Default("-").String() - dumpAllLim := dumpAll.Flag("lim", "The number of spans to transfer from the server at once."). - Default("100").Int() - graph := app.Command("graph", "Visualize span JSON as a graph.") - graphJsonFile := graph.Arg("input", "The JSON file to load").Required().String() - graphDotFile := graph.Flag("output", - "The path to write a GraphViz dotfile to. This file can be used as input to "+ - "GraphViz, in order to generate a pretty picture. See graphviz.org for more "+ - "information about generating pictures of graphs.").Default("-").String() - query := app.Command("query", "Send a query to htraced.") - queryLim := query.Flag("lim", "Maximum number of spans to retrieve.").Default("20").Int() - queryArg := query.Arg("query", "The query string to send. Query strings have the format "+ - "[TYPE] [OPERATOR] [CONST], joined by AND statements.").Required().String() - rawQuery := app.Command("rawQuery", "Send a raw JSON query to htraced.") - rawQueryArg := query.Arg("json", "The query JSON to send.").Required().String() - cmd := kingpin.MustParse(app.Parse(os.Args[1:])) - - // Add the command-line settings into the configuration. - if *addr != "" { - cnf = cnf.Clone(conf.HTRACE_WEB_ADDRESS, *addr) - } - - // Handle commands that don't require an HTrace client. - switch cmd { - case version.FullCommand(): - os.Exit(printVersion()) - case graph.FullCommand(): - err := jsonSpanFileToDotFile(*graphJsonFile, *graphDotFile) - if err != nil { - fmt.Printf("graphing error: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - os.Exit(EXIT_SUCCESS) - } - - // Create HTrace client - hcl, err := htrace.NewClient(cnf) - if err != nil { - fmt.Printf("Failed to create HTrace client: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - - // Handle commands that require an HTrace client. - switch cmd { - case version.FullCommand(): - os.Exit(printVersion()) - case serverInfo.FullCommand(): - os.Exit(printServerInfo(hcl)) - case findSpan.FullCommand(): - os.Exit(doFindSpan(hcl, common.SpanId(*findSpanId))) - case findChildren.FullCommand(): - os.Exit(doFindChildren(hcl, common.SpanId(*parentSpanId), *childLim)) - case loadJson.FullCommand(): - os.Exit(doLoadSpanJson(hcl, *loadJsonArg)) - case loadFile.FullCommand(): - os.Exit(doLoadSpanJsonFile(hcl, *loadFilePath)) - case dumpAll.FullCommand(): - err := doDumpAll(hcl, *dumpAllOutPath, *dumpAllLim) - if err != nil { - fmt.Printf("dumpAll error: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - os.Exit(EXIT_SUCCESS) - case query.FullCommand(): - err := doQueryFromString(hcl, *queryArg, *queryLim) - if err != nil { - fmt.Printf("query error: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - os.Exit(EXIT_SUCCESS) - case rawQuery.FullCommand(): - err := doRawQuery(hcl, *rawQueryArg) - if err != nil { - fmt.Printf("raw query error: %s\n", err.Error()) - os.Exit(EXIT_FAILURE) - } - os.Exit(EXIT_SUCCESS) - } - - app.UsageErrorf(os.Stderr, "You must supply a command to run.") -} - -// Print the version of the htrace binary. -func printVersion() int { - fmt.Printf("Running htrace command version %s.\n", RELEASE_VERSION) - return EXIT_SUCCESS -} - -// Print information retrieved from an htraced server via /server/info -func printServerInfo(hcl *htrace.Client) int { - info, err := hcl.GetServerInfo() - if err != nil { - fmt.Println(err.Error()) - return EXIT_FAILURE - } - fmt.Printf("HTraced server version %s (%s)\n", info.ReleaseVersion, info.GitVersion) - return EXIT_SUCCESS -} - -// Print information about a trace span. -func doFindSpan(hcl *htrace.Client, sid common.SpanId) int { - span, err := hcl.FindSpan(sid) - if err != nil { - fmt.Println(err.Error()) - return EXIT_FAILURE - } - if span == nil { - fmt.Printf("Span ID not found.\n") - return EXIT_FAILURE - } - pbuf, err := json.MarshalIndent(span, "", " ") - if err != nil { - fmt.Printf("Error: error pretty-printing span to JSON: %s\n", err.Error()) - return EXIT_FAILURE - } - fmt.Printf("%s\n", string(pbuf)) - return EXIT_SUCCESS -} - -func doLoadSpanJsonFile(hcl *htrace.Client, spanFile string) int { - if spanFile == "" { - fmt.Printf("You must specify the json file to load.\n") - return EXIT_FAILURE - } - file, err := OpenInputFile(spanFile) - if err != nil { - fmt.Printf("Failed to open %s: %s\n", spanFile, err.Error()) - return EXIT_FAILURE - } - defer file.Close() - return doLoadSpans(hcl, bufio.NewReader(file)) -} - -func doLoadSpanJson(hcl *htrace.Client, spanJson string) int { - return doLoadSpans(hcl, bytes.NewBufferString(spanJson)) -} - -func doLoadSpans(hcl *htrace.Client, reader io.Reader) int { - dec := json.NewDecoder(reader) - spans := make([]*common.Span, 0, 32) - var err error - for { - var span common.Span - if err = dec.Decode(&span); err != nil { - if err == io.EOF { - break - } - fmt.Printf("Failed to decode JSON: %s\n", err.Error()) - return EXIT_FAILURE - } - spans = append(spans, &span) - } - if *verbose { - fmt.Printf("Writing ") - prefix := "" - for i := range spans { - fmt.Printf("%s%s", prefix, spans[i].ToJson()) - prefix = ", " - } - fmt.Printf("\n") - } - err = hcl.WriteSpans(&common.WriteSpansReq{ - Spans: spans, - }) - if err != nil { - fmt.Println(err.Error()) - return EXIT_FAILURE - } - return EXIT_SUCCESS -} - -// Find information about the children of a span. -func doFindChildren(hcl *htrace.Client, sid common.SpanId, lim int) int { - spanIds, err := hcl.FindChildren(sid, lim) - if err != nil { - fmt.Printf("%s\n", err.Error()) - return EXIT_FAILURE - } - pbuf, err := json.MarshalIndent(spanIds, "", " ") - if err != nil { - fmt.Println("Error: error pretty-printing span IDs to JSON: %s", err.Error()) - return 1 - } - fmt.Printf("%s\n", string(pbuf)) - return 0 -} - -// Dump all spans from the htraced daemon. -func doDumpAll(hcl *htrace.Client, outPath string, lim int) error { - file, err := CreateOutputFile(outPath) - if err != nil { - return err - } - w := bufio.NewWriter(file) - defer func() { - if file != nil { - w.Flush() - file.Close() - } - }() - out := make(chan *common.Span, 50) - var dumpErr error - go func() { - dumpErr = hcl.DumpAll(lim, out) - }() - var numSpans int64 - nextLogTime := time.Now().Add(time.Second * 5) - for { - span, channelOpen := <-out - if !channelOpen { - break - } - if err == nil { - _, err = fmt.Fprintf(w, "%s\n", span.ToJson()) - } - if *verbose { - numSpans++ - now := time.Now() - if !now.Before(nextLogTime) { - nextLogTime = now.Add(time.Second * 5) - fmt.Printf("received %d span(s)...\n", numSpans) - } - } - } - if err != nil { - return errors.New(fmt.Sprintf("Write error %s", err.Error())) - } - if dumpErr != nil { - return errors.New(fmt.Sprintf("Dump error %s", dumpErr.Error())) - } - err = w.Flush() - if err != nil { - return err - } - err = file.Close() - file = nil - if err != nil { - return err - } - return nil -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htrace/file.go b/htrace-htraced/src/go/src/org/apache/htrace/htrace/file.go deleted file mode 100644 index ea214be..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htrace/file.go +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bufio" - "encoding/json" - "errors" - "fmt" - "io" - "org/apache/htrace/common" - "os" -) - -// A file used for input. -// Transparently supports using stdin for input. -type InputFile struct { - *os.File - path string -} - -// Open an input file. Stdin will be used when path is - -func OpenInputFile(path string) (*InputFile, error) { - if path == "-" { - return &InputFile{File: os.Stdin, path: path}, nil - } - file, err := os.Open(path) - if err != nil { - return nil, err - } - return &InputFile{File: file, path: path}, nil -} - -func (file *InputFile) Close() { - if file.path != "-" { - file.File.Close() - } -} - -// A file used for output. -// Transparently supports using stdout for output. -type OutputFile struct { - *os.File - path string -} - -// Create an output file. Stdout will be used when path is - -func CreateOutputFile(path string) (*OutputFile, error) { - if path == "-" { - return &OutputFile{File: os.Stdout, path: path}, nil - } - file, err := os.Create(path) - if err != nil { - return nil, err - } - return &OutputFile{File: file, path: path}, nil -} - -func (file *OutputFile) Close() error { - if file.path != "-" { - return file.File.Close() - } - return nil -} - -// FailureDeferringWriter is a writer which allows us to call Printf multiple -// times and then check if all the printfs succeeded at the very end, rather -// than checking after each call. We will not attempt to write more data -// after the first write failure. -type FailureDeferringWriter struct { - io.Writer - err error -} - -func NewFailureDeferringWriter(writer io.Writer) *FailureDeferringWriter { - return &FailureDeferringWriter{writer, nil} -} - -func (w *FailureDeferringWriter) Printf(format string, v ...interface{}) { - if w.err != nil { - return - } - str := fmt.Sprintf(format, v...) - _, err := w.Writer.Write([]byte(str)) - if err != nil { - w.err = err - } -} - -func (w *FailureDeferringWriter) Error() error { - return w.err -} - -// Read a file full of whitespace-separated span JSON into a slice of spans. -func readSpansFile(path string) (common.SpanSlice, error) { - file, err := OpenInputFile(path) - if err != nil { - return nil, err - } - defer file.Close() - return readSpans(bufio.NewReader(file)) -} - -// Read whitespace-separated span JSON into a slice of spans. -func readSpans(reader io.Reader) (common.SpanSlice, error) { - spans := make(common.SpanSlice, 0) - dec := json.NewDecoder(reader) - for { - var span common.Span - err := dec.Decode(&span) - if err != nil { - if err != io.EOF { - return nil, errors.New(fmt.Sprintf("Decode error after decoding %d "+ - "span(s): %s", len(spans), err.Error())) - } - break - } - spans = append(spans, &span) - } - return spans, nil -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htrace/file_test.go b/htrace-htraced/src/go/src/org/apache/htrace/htrace/file_test.go deleted file mode 100644 index b6f9cac..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htrace/file_test.go +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "errors" - "io" - "io/ioutil" - "org/apache/htrace/common" - "org/apache/htrace/conf" - "org/apache/htrace/test" - "os" - "strings" - "testing" -) - -func TestInputFileAndOutputFile(t *testing.T) { - tdir, err := ioutil.TempDir(os.TempDir(), "TestInputFileAndOutputFile") - if err != nil { - t.Fatalf("failed to create TempDir: %s\n", err.Error()) - } - defer os.RemoveAll(tdir) - tpath := tdir + conf.PATH_SEP + "test" - var ofile *OutputFile - ofile, err = CreateOutputFile(tpath) - if err != nil { - t.Fatalf("failed to create OutputFile at %s: %s\n", tpath, err.Error()) - } - defer func() { - if ofile != nil { - ofile.Close() - } - }() - w := NewFailureDeferringWriter(ofile) - w.Printf("Hello, world!\n") - w.Printf("2 + 2 = %d\n", 4) - if w.Error() != nil { - t.Fatalf("got unexpected error writing to %s: %s\n", tpath, w.Error().Error()) - } - err = ofile.Close() - ofile = nil - if err != nil { - t.Fatalf("error on closing OutputFile for %s: %s\n", tpath, err.Error()) - } - var ifile *InputFile - ifile, err = OpenInputFile(tpath) - defer ifile.Close() - expected := "Hello, world!\n2 + 2 = 4\n" - buf := make([]byte, len(expected)) - _, err = io.ReadAtLeast(ifile, buf, len(buf)) - if err != nil { - t.Fatalf("unexpected error on reading %s: %s\n", tpath, err.Error()) - } - str := string(buf) - if str != expected { - t.Fatalf("Could not read back what we wrote to %s.\n"+ - "Got:\n%s\nExpected:\n%s\n", tpath, str, expected) - } -} - -type LimitedBufferWriter struct { - buf []byte - off int -} - -const LIMITED_BUFFER_MESSAGE = "There isn't enough buffer to go around!" - -func (w *LimitedBufferWriter) Write(p []byte) (int, error) { - var nwritten int - for i := range p { - if w.off >= len(w.buf) { - return nwritten, errors.New(LIMITED_BUFFER_MESSAGE) - } - w.buf[w.off] = p[i] - w.off = w.off + 1 - nwritten++ - } - return nwritten, nil -} - -func TestFailureDeferringWriter(t *testing.T) { - lw := LimitedBufferWriter{buf: make([]byte, 20), off: 0} - w := NewFailureDeferringWriter(&lw) - w.Printf("Zippity do dah #%d\n", 1) - w.Printf("Zippity do dah #%d\n", 2) - if w.Error() == nil { - t.Fatalf("expected FailureDeferringWriter to experience a failure due to " + - "limited buffer size, but it did not.") - } - if w.Error().Error() != LIMITED_BUFFER_MESSAGE { - t.Fatalf("expected FailureDeferringWriter to have the error message %s, but "+ - "the message was %s\n", LIMITED_BUFFER_MESSAGE, w.Error().Error()) - } - expected := "Zippity do dah #1\nZi" - if string(lw.buf) != expected { - t.Fatalf("expected LimitedBufferWriter to contain %s, but it contained %s "+ - "instead.\n", expected, string(lw.buf)) - } -} - -func TestReadSpans(t *testing.T) { - SPAN_TEST_STR := `{"i":"bdd6d4ee48de59bf","s":"c0681027d3ea4928",` + - `"b":1424736225037,"e":1424736225901,"d":"ClientNamenodeProtocol#getFileInfo",` + - `"r":"FsShell","p":["60538dfb4df91418"]} -{"i":"bdd6d4ee48de59bf","s":"60538dfb4df91418","b":1424736224969,` + - `"e":1424736225960,"d":"getFileInfo","r":"FsShell","p":[],"n":{"path":"/"}} -` - r := strings.NewReader(SPAN_TEST_STR) - spans, err := readSpans(r) - if err != nil { - t.Fatalf("Failed to read spans from string via readSpans: %s\n", err.Error()) - } - SPAN_TEST_EXPECTED := common.SpanSlice{ - &common.Span{ - Id: test.SpanId("c0681027d3ea4928"), - SpanData: common.SpanData{ - TraceId: test.SpanId("bdd6d4ee48de59bf"), - Begin: 1424736225037, - End: 1424736225901, - Description: "ClientNamenodeProtocol#getFileInfo", - ProcessId: "FsShell", - Parents: []common.SpanId{test.SpanId("60538dfb4df91418")}, - }, - }, - &common.Span{ - Id: test.SpanId("60538dfb4df91418"), - SpanData: common.SpanData{ - TraceId: test.SpanId("bdd6d4ee48de59bf"), - Begin: 1424736224969, - End: 1424736225960, - Description: "getFileInfo", - ProcessId: "FsShell", - Parents: []common.SpanId{}, - Info: common.TraceInfoMap{ - "path": "/", - }, - }, - }, - } - if len(spans) != len(SPAN_TEST_EXPECTED) { - t.Fatalf("Expected %d spans, but got %d\n", - len(SPAN_TEST_EXPECTED), len(spans)) - } - for i := range SPAN_TEST_EXPECTED { - common.ExpectSpansEqual(t, spans[i], SPAN_TEST_EXPECTED[i]) - } -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htrace/graph.go b/htrace-htraced/src/go/src/org/apache/htrace/htrace/graph.go deleted file mode 100644 index dabf2df..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htrace/graph.go +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bufio" - "errors" - "fmt" - "io" - "org/apache/htrace/common" - "os" - "sort" -) - -// Create a dotfile from a json file. -func jsonSpanFileToDotFile(jsonFile string, dotFile string) error { - spans, err := readSpansFile(jsonFile) - if err != nil { - return errors.New(fmt.Sprintf("error reading %s: %s", - jsonFile, err.Error())) - } - var file *OutputFile - file, err = CreateOutputFile(dotFile) - if err != nil { - return errors.New(fmt.Sprintf("error opening %s for write: %s", - dotFile, err.Error())) - } - defer func() { - if file != nil { - file.Close() - } - }() - writer := bufio.NewWriter(file) - err = spansToDot(spans, writer) - if err != nil { - return err - } - err = writer.Flush() - if err != nil { - return err - } - err = file.Close() - file = nil - return err -} - -// Create output in dotfile format from a set of spans. -func spansToDot(spans common.SpanSlice, writer io.Writer) error { - sort.Sort(spans) - idMap := make(map[common.SpanId]*common.Span) - for i := range spans { - span := spans[i] - if idMap[span.Id] != nil { - fmt.Fprintf(os.Stderr, "There were multiple spans listed which "+ - "had ID %s.\nFirst:%s\nOther:%s\n", span.Id.String(), - idMap[span.Id].ToJson(), span.ToJson()) - } else { - idMap[span.Id] = span - } - } - childMap := make(map[common.SpanId]common.SpanSlice) - for i := range spans { - child := spans[i] - for j := range child.Parents { - parent := idMap[child.Parents[j]] - if parent == nil { - fmt.Fprintf(os.Stderr, "Can't find parent id %s for %s\n", - child.Parents[j].String(), child.ToJson()) - } else { - children := childMap[parent.Id] - if children == nil { - children = make(common.SpanSlice, 0) - } - children = append(children, child) - childMap[parent.Id] = children - } - } - } - w := NewFailureDeferringWriter(writer) - w.Printf("digraph spans {\n") - // Write out the nodes with their descriptions. - for i := range spans { - w.Printf(fmt.Sprintf(` "%s" [label="%s"];`+"\n", - spans[i].Id.String(), spans[i].Description)) - } - // Write out the edges between nodes... the parent/children relationships - for i := range spans { - children := childMap[spans[i].Id] - sort.Sort(children) - if children != nil { - for c := range children { - w.Printf(fmt.Sprintf(` "%s" -> "%s";`+"\n", - spans[i].Id.String(), children[c].Id)) - } - } - } - w.Printf("}\n") - return w.Error() -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htrace/graph_test.go b/htrace-htraced/src/go/src/org/apache/htrace/htrace/graph_test.go deleted file mode 100644 index 8698a98..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htrace/graph_test.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bytes" - "org/apache/htrace/common" - "org/apache/htrace/test" - "testing" -) - -func TestSpansToDot(t *testing.T) { - TEST_SPANS := common.SpanSlice{ - &common.Span{ - Id: test.SpanId("6af3cc058e5d829d"), - SpanData: common.SpanData{ - TraceId: test.SpanId("0e4716fe911244de"), - Begin: 1424813349020, - End: 1424813349134, - Description: "newDFSInputStream", - ProcessId: "FsShell", - Parents: []common.SpanId{}, - Info: common.TraceInfoMap{ - "path": "/", - }, - }, - }, - &common.Span{ - Id: test.SpanId("75d16cc5b2c07d8a"), - SpanData: common.SpanData{ - TraceId: test.SpanId("0e4716fe911244de"), - Begin: 1424813349025, - End: 1424813349133, - Description: "getBlockLocations", - ProcessId: "FsShell", - Parents: []common.SpanId{test.SpanId("6af3cc058e5d829d")}, - }, - }, - &common.Span{ - Id: test.SpanId("e2c7273efb280a8c"), - SpanData: common.SpanData{ - TraceId: test.SpanId("0e4716fe911244de"), - Begin: 1424813349027, - End: 1424813349073, - Description: "ClientNamenodeProtocol#getBlockLocations", - ProcessId: "FsShell", - Parents: []common.SpanId{test.SpanId("75d16cc5b2c07d8a")}, - }, - }, - } - w := bytes.NewBuffer(make([]byte, 0, 2048)) - err := spansToDot(TEST_SPANS, w) - if err != nil { - t.Fatalf("spansToDot failed: error %s\n", err.Error()) - } - EXPECTED_STR := `digraph spans { - "6af3cc058e5d829d" [label="newDFSInputStream"]; - "75d16cc5b2c07d8a" [label="getBlockLocations"]; - "e2c7273efb280a8c" [label="ClientNamenodeProtocol#getBlockLocations"]; - "6af3cc058e5d829d" -> "75d16cc5b2c07d8a"; - "75d16cc5b2c07d8a" -> "e2c7273efb280a8c"; -} -` - if w.String() != EXPECTED_STR { - t.Fatalf("Expected to get:\n%s\nGot:\n%s\n", EXPECTED_STR, w.String()) - } -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htrace/queries.go b/htrace-htraced/src/go/src/org/apache/htrace/htrace/queries.go deleted file mode 100644 index 4ff246c..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htrace/queries.go +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "encoding/json" - "errors" - "fmt" - htrace "org/apache/htrace/client" - "org/apache/htrace/common" - "strings" - "unicode" -) - -// Convert a string into a whitespace-separated sequence of strings. -func tokenize(str string) []string { - prevQuote := rune(0) - f := func(c rune) bool { - switch { - case c == prevQuote: - prevQuote = rune(0) - return false - case prevQuote != rune(0): - return false - case unicode.In(c, unicode.Quotation_Mark): - prevQuote = c - return false - default: - return unicode.IsSpace(c) - } - } - return strings.FieldsFunc(str, f) -} - -// Parses a query string in the format of a series of -// [TYPE] [OPERATOR] [CONST] tuples, joined by AND statements. -type predicateParser struct { - tokens []string - curToken int -} - -func (ps *predicateParser) Parse() (*common.Predicate, error) { - if ps.curToken > len(ps.tokens) { - return nil, nil - } - if ps.curToken > 0 { - if strings.ToLower(ps.tokens[ps.curToken]) != "and" { - return nil, errors.New(fmt.Sprintf("Error parsing on token %d: "+ - "expected predicates to be joined by 'and', but found '%s'", - ps.curToken, ps.tokens[ps.curToken])) - } - ps.curToken++ - if ps.curToken > len(ps.tokens) { - return nil, errors.New(fmt.Sprintf("Nothing found after 'and' at "+ - "token %d", ps.curToken)) - } - } - field := common.Field(ps.tokens[ps.curToken]) - if !field.IsValid() { - return nil, errors.New(fmt.Sprintf("Invalid field specifier at token %d. "+ - "Can't understand %s. Valid field specifiers are %v", ps.curToken, - ps.tokens[ps.curToken], common.ValidFields())) - } - ps.curToken++ - if ps.curToken > len(ps.tokens) { - return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ - "at token %d", ps.curToken)) - } - op := common.Op(ps.tokens[ps.curToken]) - if !op.IsValid() { - return nil, errors.New(fmt.Sprintf("Invalid operation specifier at token %d. "+ - "Can't understand %s. Valid operation specifiers are %v", ps.curToken, - ps.tokens[ps.curToken], common.ValidOps())) - } - ps.curToken++ - if ps.curToken > len(ps.tokens) { - return nil, errors.New(fmt.Sprintf("Nothing found after field specifier "+ - "at token %d", ps.curToken)) - } - val := ps.tokens[ps.curToken] - return &common.Predicate{Op: op, Field: field, Val: val}, nil -} - -func parseQueryString(str string) ([]common.Predicate, error) { - ps := predicateParser{tokens: tokenize(str)} - preds := make([]common.Predicate, 0) - for { - pred, err := ps.Parse() - if pred == nil { - break - } - if err != nil { - return nil, err - } - } - if len(preds) == 0 { - return nil, errors.New("Empty query string") - } - return preds, nil -} - -// Send a query from a query string. -func doQueryFromString(hcl *htrace.Client, str string, lim int) error { - query := &common.Query{Lim: lim} - var err error - query.Predicates, err = parseQueryString(str) - if err != nil { - return err - } - return doQuery(hcl, query) -} - -// Send a query from a raw JSON string. -func doRawQuery(hcl *htrace.Client, str string) error { - jsonBytes := []byte(str) - var query common.Query - err := json.Unmarshal(jsonBytes, &query) - if err != nil { - return errors.New(fmt.Sprintf("Error parsing provided JSON: %s\n", err.Error())) - } - return doQuery(hcl, &query) -} - -// Send a query. -func doQuery(hcl *htrace.Client, query *common.Query) error { - if *verbose { - qbytes, err := json.Marshal(*query) - if err != nil { - qbytes = []byte("marshaling error: " + err.Error()) - } - fmt.Printf("Sending query: %s\n", string(qbytes)) - } - spans, err := hcl.Query(query) - if err != nil { - return err - } - if *verbose { - fmt.Printf("%d results...\n", len(spans)) - } - for i := range spans { - fmt.Printf("%s\n", spans[i].ToJson()) - } - return nil -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htraced/client_test.go b/htrace-htraced/src/go/src/org/apache/htrace/htraced/client_test.go deleted file mode 100644 index 218c1c8..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htraced/client_test.go +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "fmt" - "math/rand" - htrace "org/apache/htrace/client" - "org/apache/htrace/common" - "org/apache/htrace/test" - "sort" - "testing" - "time" -) - -func TestClientGetServerInfo(t *testing.T) { - htraceBld := &MiniHTracedBuilder{Name: "TestClientGetServerInfo", - DataDirs: make([]string, 2)} - ht, err := htraceBld.Build() - if err != nil { - t.Fatalf("failed to create datastore: %s", err.Error()) - } - defer ht.Close() - var hcl *htrace.Client - hcl, err = htrace.NewClient(ht.ClientConf()) - if err != nil { - t.Fatalf("failed to create client: %s", err.Error()) - } - _, err = hcl.GetServerInfo() - if err != nil { - t.Fatalf("failed to call GetServerInfo: %s", err.Error()) - } -} - -func createRandomTestSpans(amount int) common.SpanSlice { - rnd := rand.New(rand.NewSource(2)) - allSpans := make(common.SpanSlice, amount) - allSpans[0] = test.NewRandomSpan(rnd, allSpans[0:0]) - for i := 1; i < amount; i++ { - allSpans[i] = test.NewRandomSpan(rnd, allSpans[1:i]) - } - allSpans[1].SpanData.Parents = []common.SpanId{common.SpanId(allSpans[0].Id)} - return allSpans -} - -func TestClientOperations(t *testing.T) { - htraceBld := &MiniHTracedBuilder{Name: "TestClientOperations", - DataDirs: make([]string, 2)} - ht, err := htraceBld.Build() - if err != nil { - t.Fatalf("failed to create datastore: %s", err.Error()) - } - defer ht.Close() - var hcl *htrace.Client - hcl, err = htrace.NewClient(ht.ClientConf()) - if err != nil { - t.Fatalf("failed to create client: %s", err.Error()) - } - - // Create some random trace spans. - NUM_TEST_SPANS := 30 - allSpans := createRandomTestSpans(NUM_TEST_SPANS) - - // Write half of the spans to htraced via the client. - err = hcl.WriteSpans(&common.WriteSpansReq{ - Spans: allSpans[0 : NUM_TEST_SPANS/2], - }) - if err != nil { - t.Fatalf("WriteSpans(0:%d) failed: %s\n", NUM_TEST_SPANS/2, - err.Error()) - } - - // Look up the first half of the spans. They should be found. - var span *common.Span - for i := 0; i < NUM_TEST_SPANS/2; i++ { - span, err = hcl.FindSpan(allSpans[i].Id) - if err != nil { - t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) - } - common.ExpectSpansEqual(t, allSpans[i], span) - } - - // Look up the second half of the spans. They should not be found. - for i := NUM_TEST_SPANS / 2; i < NUM_TEST_SPANS; i++ { - span, err = hcl.FindSpan(allSpans[i].Id) - if err != nil { - t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) - } - if span != nil { - t.Fatalf("Unexpectedly found a span we never write to "+ - "the server: FindSpan(%d) succeeded\n", i) - } - } - - // Test FindChildren - childSpan := allSpans[1] - parentId := childSpan.Parents[0] - var children []common.SpanId - children, err = hcl.FindChildren(parentId, 1) - if err != nil { - t.Fatalf("FindChildren(%s) failed: %s\n", parentId, err.Error()) - } - if len(children) != 1 { - t.Fatalf("FindChildren(%s) returned an invalid number of "+ - "children: expected %d, got %d\n", parentId, 1, len(children)) - } - if children[0] != childSpan.Id { - t.Fatalf("FindChildren(%s) returned an invalid child id: expected %s, "+ - " got %s\n", parentId, childSpan.Id, children[0]) - } - - // Test FindChildren on a span that has no children - childlessSpan := allSpans[NUM_TEST_SPANS/2] - children, err = hcl.FindChildren(childlessSpan.Id, 10) - if err != nil { - t.Fatalf("FindChildren(%d) failed: %s\n", childlessSpan.Id, err.Error()) - } - if len(children) != 0 { - t.Fatalf("FindChildren(%d) returned an invalid number of "+ - "children: expected %d, got %d\n", childlessSpan.Id, 0, len(children)) - } - - // Test Query - var query common.Query - query = common.Query{Lim: 10} - spans, err := hcl.Query(&query) - if err != nil { - t.Fatalf("Query({lim: %d}) failed: %s\n", 10, err.Error()) - } - if len(spans) != 10 { - t.Fatalf("Query({lim: %d}) returned an invalid number of "+ - "children: expected %d, got %d\n", 10, 10, len(spans)) - } -} - -func TestDumpAll(t *testing.T) { - htraceBld := &MiniHTracedBuilder{Name: "TestDumpAll", - DataDirs: make([]string, 2)} - ht, err := htraceBld.Build() - if err != nil { - t.Fatalf("failed to create datastore: %s", err.Error()) - } - defer ht.Close() - var hcl *htrace.Client - hcl, err = htrace.NewClient(ht.ClientConf()) - if err != nil { - t.Fatalf("failed to create client: %s", err.Error()) - } - - NUM_TEST_SPANS := 100 - allSpans := createRandomTestSpans(NUM_TEST_SPANS) - sort.Sort(allSpans) - err = hcl.WriteSpans(&common.WriteSpansReq{ - Spans: allSpans, - }) - if err != nil { - t.Fatalf("WriteSpans failed: %s\n", err.Error()) - } - out := make(chan *common.Span, 50) - var dumpErr error - go func() { - dumpErr = hcl.DumpAll(3, out) - }() - var numSpans int - nextLogTime := time.Now().Add(time.Millisecond * 5) - for { - span, channelOpen := <-out - if !channelOpen { - break - } - common.ExpectSpansEqual(t, allSpans[numSpans], span) - numSpans++ - if testing.Verbose() { - now := time.Now() - if !now.Before(nextLogTime) { - nextLogTime = now - nextLogTime = nextLogTime.Add(time.Millisecond * 5) - fmt.Printf("read back %d span(s)...\n", numSpans) - } - } - } - if numSpans != len(allSpans) { - t.Fatalf("expected to read %d spans... but only read %d\n", - len(allSpans), numSpans) - } - if dumpErr != nil { - t.Fatalf("got dump error %s\n", dumpErr.Error()) - } -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htraced/datastore.go b/htrace-htraced/src/go/src/org/apache/htrace/htraced/datastore.go deleted file mode 100644 index 0742555..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htraced/datastore.go +++ /dev/null @@ -1,999 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bytes" - "encoding/gob" - "errors" - "fmt" - "github.com/jmhodges/levigo" - "org/apache/htrace/common" - "org/apache/htrace/conf" - "os" - "strconv" - "strings" - "sync/atomic" -) - -// -// The data store code for HTraced. -// -// This code stores the trace spans. We use levelDB here so that we don't have to store everything -// in memory at all times. The data is sharded across multiple levelDB databases in multiple -// directories. Normally, these multiple directories will be on multiple disk drives. -// -// The main emphasis in the HTraceD data store is on quickly and efficiently storing trace span data -// coming from many daemons. Durability is not as big a concern as in some data stores, since -// losing a little bit of trace data if htraced goes down is not critical. We use the "gob" package -// for serialization. We assume that there will be many more writes than reads. -// -// Schema -// m -> dataStoreVersion -// s[8-byte-big-endian-sid] -> SpanData -// b[8-byte-big-endian-begin-time][8-byte-big-endian-child-sid] -> {} -// e[8-byte-big-endian-end-time][8-byte-big-endian-child-sid] -> {} -// d[8-byte-big-endian-duration][8-byte-big-endian-child-sid] -> {} -// p[8-byte-big-endian-parent-sid][8-byte-big-endian-child-sid] -> {} -// -// Note that span IDs are unsigned 64-bit numbers. -// Begin times, end times, and durations are signed 64-bit numbers. -// In order to get LevelDB to properly compare the signed 64-bit quantities, -// we flip the highest bit. This way, we can get leveldb to view negative -// quantities as less than non-negative ones. This also means that we can do -// all queries using unsigned 64-bit math, rather than having to special-case -// the signed fields. -// - -const UNKNOWN_LAYOUT_VERSION = 0 -const CURRENT_LAYOUT_VERSION = 2 - -var EMPTY_BYTE_BUF []byte = []byte{} - -const VERSION_KEY = 'v' -const SPAN_ID_INDEX_PREFIX = 's' -const BEGIN_TIME_INDEX_PREFIX = 'b' -const END_TIME_INDEX_PREFIX = 'e' -const DURATION_INDEX_PREFIX = 'd' -const PARENT_ID_INDEX_PREFIX = 'p' -const INVALID_INDEX_PREFIX = 0 - -type Statistics struct { - NumSpansWritten uint64 -} - -func (stats *Statistics) IncrementWrittenSpans() { - atomic.AddUint64(&stats.NumSpansWritten, 1) -} - -// Make a copy of the statistics structure, using atomic operations. -func (stats *Statistics) Copy() *Statistics { - return &Statistics{ - NumSpansWritten: atomic.LoadUint64(&stats.NumSpansWritten), - } -} - -// Translate an 8-byte value into a leveldb key. -func makeKey(tag byte, val uint64) []byte { - return []byte{ - tag, - byte(0xff & (val >> 56)), - byte(0xff & (val >> 48)), - byte(0xff & (val >> 40)), - byte(0xff & (val >> 32)), - byte(0xff & (val >> 24)), - byte(0xff & (val >> 16)), - byte(0xff & (val >> 8)), - byte(0xff & (val >> 0)), - } -} - -func keyToInt(key []byte) uint64 { - var id uint64 - id = (uint64(key[0]) << 56) | - (uint64(key[1]) << 48) | - (uint64(key[2]) << 40) | - (uint64(key[3]) << 32) | - (uint64(key[4]) << 24) | - (uint64(key[5]) << 16) | - (uint64(key[6]) << 8) | - (uint64(key[7]) << 0) - return id -} - -func makeSecondaryKey(tag byte, fir uint64, sec uint64) []byte { - return []byte{ - tag, - byte(0xff & (fir >> 56)), - byte(0xff & (fir >> 48)), - byte(0xff & (fir >> 40)), - byte(0xff & (fir >> 32)), - byte(0xff & (fir >> 24)), - byte(0xff & (fir >> 16)), - byte(0xff & (fir >> 8)), - byte(0xff & (fir >> 0)), - byte(0xff & (sec >> 56)), - byte(0xff & (sec >> 48)), - byte(0xff & (sec >> 40)), - byte(0xff & (sec >> 32)), - byte(0xff & (sec >> 24)), - byte(0xff & (sec >> 16)), - byte(0xff & (sec >> 8)), - byte(0xff & (sec >> 0)), - } -} - -// A single directory containing a levelDB instance. -type shard struct { - // The data store that this shard is part of - store *dataStore - - // The LevelDB instance. - ldb *levigo.DB - - // The path to the leveldb directory this shard is managing. - path string - - // Incoming requests to write Spans. - incoming chan *common.Span - - // The channel we will send a bool to when we exit. - exited chan bool -} - -// Process incoming spans for a shard. -func (shd *shard) processIncoming() { - lg := shd.store.lg - for { - span := <-shd.incoming - if span == nil { - lg.Infof("Shard processor for %s exiting.\n", shd.path) - shd.exited <- true - return - } - err := shd.writeSpan(span) - if err != nil { - lg.Errorf("Shard processor for %s got fatal error %s.\n", shd.path, err.Error()) - } else { - lg.Tracef("Shard processor for %s wrote span %s.\n", shd.path, span.ToJson()) - } - } -} - -// Convert a signed 64-bit number into an unsigned 64-bit number. We flip the -// highest bit, so that negative input values map to unsigned numbers which are -// less than non-negative input values. -func s2u64(val int64) uint64 { - ret := uint64(val) - ret ^= 0x8000000000000000 - return ret -} - -func (shd *shard) writeSpan(span *common.Span) error { - batch := levigo.NewWriteBatch() - defer batch.Close() - - // Add SpanData to batch. - spanDataBuf := new(bytes.Buffer) - spanDataEnc := gob.NewEncoder(spanDataBuf) - err := spanDataEnc.Encode(span.SpanData) - if err != nil { - return err - } - batch.Put(makeKey(SPAN_ID_INDEX_PREFIX, span.Id.Val()), spanDataBuf.Bytes()) - - // Add this to the parent index. - for parentIdx := range span.Parents { - batch.Put(makeSecondaryKey(PARENT_ID_INDEX_PREFIX, - span.Parents[parentIdx].Val(), span.Id.Val()), EMPTY_BYTE_BUF) - } - - // Add to the other secondary indices. - batch.Put(makeSecondaryKey(BEGIN_TIME_INDEX_PREFIX, s2u64(span.Begin), - span.Id.Val()), EMPTY_BYTE_BUF) - batch.Put(makeSecondaryKey(END_TIME_INDEX_PREFIX, s2u64(span.End), - span.Id.Val()), EMPTY_BYTE_BUF) - batch.Put(makeSecondaryKey(DURATION_INDEX_PREFIX, s2u64(span.Duration()), - span.Id.Val()), EMPTY_BYTE_BUF) - - err = shd.ldb.Write(shd.store.writeOpts, batch) - if err != nil { - return err - } - shd.store.stats.IncrementWrittenSpans() - if shd.store.WrittenSpans != nil { - shd.store.WrittenSpans <- span - } - return nil -} - -func (shd *shard) FindChildren(sid common.SpanId, childIds []common.SpanId, - lim int32) ([]common.SpanId, int32, error) { - searchKey := makeKey('p', sid.Val()) - iter := shd.ldb.NewIterator(shd.store.readOpts) - defer iter.Close() - iter.Seek(searchKey) - for { - if !iter.Valid() { - break - } - if lim == 0 { - break - } - key := iter.Key() - if !bytes.HasPrefix(key, searchKey) { - break - } - id := common.SpanId(keyToInt(key[9:])) - childIds = append(childIds, id) - lim-- - iter.Next() - } - return childIds, lim, nil -} - -// Close a shard. -func (shd *shard) Close() { - lg := shd.store.lg - shd.incoming <- nil - lg.Infof("Waiting for %s to exit...\n", shd.path) - if shd.exited != nil { - <-shd.exited - } - shd.ldb.Close() - lg.Infof("Closed %s...\n", shd.path) -} - -// The Data Store. -type dataStore struct { - lg *common.Logger - - // The shards which manage our LevelDB instances. - shards []*shard - - // I/O statistics for all shards. - stats Statistics - - // The read options to use for LevelDB. - readOpts *levigo.ReadOptions - - // The write options to use for LevelDB. - writeOpts *levigo.WriteOptions - - // If non-null, a channel we will send spans to once we finish writing them. This is only used - // for testing. - WrittenSpans chan *common.Span -} - -func CreateDataStore(cnf *conf.Config, writtenSpans chan *common.Span) (*dataStore, error) { - // Get the configuration. - clearStored := cnf.GetBool(conf.HTRACE_DATA_STORE_CLEAR) - dirsStr := cnf.Get(conf.HTRACE_DATA_STORE_DIRECTORIES) - dirs := strings.Split(dirsStr, conf.PATH_LIST_SEP) - - var err error - lg := common.NewLogger("datastore", cnf) - store := &dataStore{lg: lg, shards: []*shard{}, WrittenSpans: writtenSpans} - - // If we return an error, close the store. - defer func() { - if err != nil { - store.Close() - store = nil - } - }() - - store.readOpts = levigo.NewReadOptions() - store.readOpts.SetFillCache(true) - store.writeOpts = levigo.NewWriteOptions() - store.writeOpts.SetSync(false) - - // Open all shards - for idx := range dirs { - path := dirs[idx] + conf.PATH_SEP + "db" - var shd *shard - shd, err = CreateShard(store, cnf, path, clearStored) - if err != nil { - lg.Errorf("Error creating shard %s: %s\n", path, err.Error()) - return nil, err - } - store.shards = append(store.shards, shd) - } - for idx := range store.shards { - shd := store.shards[idx] - shd.exited = make(chan bool, 1) - go shd.processIncoming() - } - return store, nil -} - -func CreateShard(store *dataStore, cnf *conf.Config, path string, - clearStored bool) (*shard, error) { - lg := store.lg - if clearStored { - fi, err := os.Stat(path) - if err != nil && !os.IsNotExist(err) { - lg.Errorf("Failed to stat %s: %s\n", path, err.Error()) - return nil, err - } - if fi != nil { - err = os.RemoveAll(path) - if err != nil { - lg.Errorf("Failed to clear existing datastore directory %s: %s\n", - path, err.Error()) - return nil, err - } - lg.Infof("Cleared existing datastore directory %s\n", path) - } - } - err := os.MkdirAll(path, 0777) - if err != nil { - lg.Errorf("Failed to MkdirAll(%s): %s\n", path, err.Error()) - return nil, err - } - var shd *shard - openOpts := levigo.NewOptions() - defer openOpts.Close() - newlyCreated := false - ldb, err := levigo.Open(path, openOpts) - if err == nil { - store.lg.Infof("LevelDB opened %s\n", path) - } else { - store.lg.Debugf("LevelDB failed to open %s: %s\n", path, err.Error()) - openOpts.SetCreateIfMissing(true) - ldb, err = levigo.Open(path, openOpts) - if err != nil { - store.lg.Errorf("LevelDB failed to create %s: %s\n", path, err.Error()) - return nil, err - } - store.lg.Infof("Created new LevelDB instance in %s\n", path) - newlyCreated = true - } - defer func() { - if shd == nil { - ldb.Close() - } - }() - lv, err := readLayoutVersion(store, ldb) - if err != nil { - store.lg.Errorf("Got error while reading datastore version for %s: %s\n", - path, err.Error()) - return nil, err - } - if newlyCreated && (lv == UNKNOWN_LAYOUT_VERSION) { - err = writeDataStoreVersion(store, ldb, CURRENT_LAYOUT_VERSION) - if err != nil { - store.lg.Errorf("Got error while writing datastore version for %s: %s\n", - path, err.Error()) - return nil, err - } - store.lg.Tracef("Wrote layout version %d to shard at %s.\n", - CURRENT_LAYOUT_VERSION, path) - } else if lv != CURRENT_LAYOUT_VERSION { - versionName := "unknown" - if lv != UNKNOWN_LAYOUT_VERSION { - versionName = fmt.Sprintf("%d", lv) - } - store.lg.Errorf("Can't read old datastore. Its layout version is %s, but this "+ - "software is at layout version %d. Please set %s to clear the datastore "+ - "on startup, or clear it manually.\n", versionName, - CURRENT_LAYOUT_VERSION, conf.HTRACE_DATA_STORE_CLEAR) - return nil, errors.New(fmt.Sprintf("Invalid layout version: got %s, expected %d.", - versionName, CURRENT_LAYOUT_VERSION)) - } else { - store.lg.Tracef("Found layout version %d in %s.\n", lv, path) - } - spanBufferSize := cnf.GetInt(conf.HTRACE_DATA_STORE_SPAN_BUFFER_SIZE) - shd = &shard{store: store, ldb: ldb, path: path, - incoming: make(chan *common.Span, spanBufferSize)} - return shd, nil -} - -// Read the datastore version of a leveldb instance. -func readLayoutVersion(store *dataStore, ldb *levigo.DB) (uint32, error) { - buf, err := ldb.Get(store.readOpts, []byte{VERSION_KEY}) - if err != nil { - return 0, err - } - if len(buf) == 0 { - return 0, nil - } - r := bytes.NewBuffer(buf) - decoder := gob.NewDecoder(r) - var v uint32 - err = decoder.Decode(&v) - if err != nil { - return 0, err - } - return v, nil -} - -// Write the datastore version to a shard. -func writeDataStoreVersion(store *dataStore, ldb *levigo.DB, v uint32) error { - w := new(bytes.Buffer) - encoder := gob.NewEncoder(w) - err := encoder.Encode(&v) - if err != nil { - return err - } - return ldb.Put(store.writeOpts, []byte{VERSION_KEY}, w.Bytes()) -} - -func (store *dataStore) GetStatistics() *Statistics { - return store.stats.Copy() -} - -// Close the DataStore. -func (store *dataStore) Close() { - for idx := range store.shards { - store.shards[idx].Close() - store.shards[idx] = nil - } - if store.readOpts != nil { - store.readOpts.Close() - store.readOpts = nil - } - if store.writeOpts != nil { - store.writeOpts.Close() - store.writeOpts = nil - } - if store.lg != nil { - store.lg.Close() - store.lg = nil - } -} - -// Get the index of the shard which stores the given spanId. -func (store *dataStore) getShardIndex(sid common.SpanId) int { - return int(sid.Val() % uint64(len(store.shards))) -} - -func (store *dataStore) WriteSpan(span *common.Span) { - store.shards[store.getShardIndex(span.Id)].incoming <- span -} - -func (store *dataStore) FindSpan(sid common.SpanId) *common.Span { - return store.shards[store.getShardIndex(sid)].FindSpan(sid) -} - -func (shd *shard) FindSpan(sid common.SpanId) *common.Span { - lg := shd.store.lg - buf, err := shd.ldb.Get(shd.store.readOpts, makeKey('s', sid.Val())) - if err != nil { - if strings.Index(err.Error(), "NotFound:") != -1 { - return nil - } - lg.Warnf("Shard(%s): FindSpan(%s) error: %s\n", - shd.path, sid.String(), err.Error()) - return nil - } - var span *common.Span - span, err = shd.decodeSpan(sid, buf) - if err != nil { - lg.Errorf("Shard(%s): FindSpan(%s) decode error: %s\n", - shd.path, sid.String(), err.Error()) - return nil - } - return span -} - -func (shd *shard) decodeSpan(sid common.SpanId, buf []byte) (*common.Span, error) { - r := bytes.NewBuffer(buf) - decoder := gob.NewDecoder(r) - data := common.SpanData{} - err := decoder.Decode(&data) - if err != nil { - return nil, err - } - // Gob encoding translates empty slices to nil. Reverse this so that we're always dealing with - // non-nil slices. - if data.Parents == nil { - data.Parents = []common.SpanId{} - } - return &common.Span{Id: common.SpanId(sid), SpanData: data}, nil -} - -// Find the children of a given span id. -func (store *dataStore) FindChildren(sid common.SpanId, lim int32) []common.SpanId { - childIds := make([]common.SpanId, 0) - var err error - - startIdx := store.getShardIndex(sid) - idx := startIdx - numShards := len(store.shards) - for { - if lim == 0 { - break - } - shd := store.shards[idx] - childIds, lim, err = shd.FindChildren(sid, childIds, lim) - if err != nil { - store.lg.Errorf("Shard(%s): FindChildren(%s) error: %s\n", - shd.path, sid.String(), err.Error()) - } - idx++ - if idx >= numShards { - idx = 0 - } - if idx == startIdx { - break - } - } - return childIds -} - -type predicateData struct { - *common.Predicate - uintKey uint64 - strKey string -} - -func loadPredicateData(pred *common.Predicate) (*predicateData, error) { - p := predicateData{Predicate: pred} - - // Parse the input value given to make sure it matches up with the field - // type. - switch pred.Field { - case common.SPAN_ID: - // Span IDs are sent as hex strings. - var id common.SpanId - if err := id.FromString(pred.Val); err != nil { - return nil, errors.New(fmt.Sprintf("Unable to parse span id '%s': %s", - pred.Val, err.Error())) - } - p.uintKey = id.Val() - break - case common.DESCRIPTION: - // Any string is valid for a description. - p.strKey = pred.Val - break - case common.BEGIN_TIME, common.END_TIME, common.DURATION: - // Parse a base-10 signed numeric field. - v, err := strconv.ParseInt(pred.Val, 10, 64) - if err != nil { - return nil, errors.New(fmt.Sprintf("Unable to parse %s '%s': %s", - pred.Field, pred.Val, err.Error())) - } - p.uintKey = s2u64(v) - break - default: - return nil, errors.New(fmt.Sprintf("Unknown field %s", pred.Field)) - } - - // Validate the predicate operation. - switch pred.Op { - case common.EQUALS, common.LESS_THAN_OR_EQUALS, - common.GREATER_THAN_OR_EQUALS, common.GREATER_THAN: - break - case common.CONTAINS: - if p.fieldIsNumeric() { - return nil, errors.New(fmt.Sprintf("Can't use CONTAINS on a "+ - "numeric field like '%s'", pred.Field)) - } - default: - return nil, errors.New(fmt.Sprintf("Unknown predicate operation '%s'", - pred.Op)) - } - - return &p, nil -} - -// Get the index prefix for this predicate, or 0 if it is not indexed. -func (pred *predicateData) getIndexPrefix() byte { - switch pred.Field { - case common.SPAN_ID: - return SPAN_ID_INDEX_PREFIX - case common.BEGIN_TIME: - return BEGIN_TIME_INDEX_PREFIX - case common.END_TIME: - return END_TIME_INDEX_PREFIX - case common.DURATION: - return DURATION_INDEX_PREFIX - default: - return INVALID_INDEX_PREFIX - } -} - -// Returns true if the predicate type is numeric. -func (pred *predicateData) fieldIsNumeric() bool { - switch pred.Field { - case common.SPAN_ID, common.BEGIN_TIME, common.END_TIME, common.DURATION: - return true - default: - return false - } -} - -// Get the values that this predicate cares about for a given span. -func (pred *predicateData) extractRelevantSpanData(span *common.Span) (uint64, string) { - switch pred.Field { - case common.SPAN_ID: - return span.Id.Val(), "" - case common.DESCRIPTION: - return 0, span.Description - case common.BEGIN_TIME: - return s2u64(span.Begin), "" - case common.END_TIME: - return s2u64(span.End), "" - case common.DURATION: - return s2u64(span.Duration()), "" - default: - panic(fmt.Sprintf("Field type %s isn't a 64-bit integer.", pred.Field)) - } -} - -func (pred *predicateData) spanPtrIsBefore(a *common.Span, b *common.Span) bool { - // nil is after everything. - if a == nil { - if b == nil { - return false - } - return false - } else if b == nil { - return true - } - // Compare the spans according to this predicate. - aInt, aStr := pred.extractRelevantSpanData(a) - bInt, bStr := pred.extractRelevantSpanData(b) - if pred.fieldIsNumeric() { - if pred.Op.IsDescending() { - return aInt > bInt - } else { - return aInt < bInt - } - } else { - if pred.Op.IsDescending() { - return aStr > bStr - } else { - return aStr < bStr - } - } -} - -// Returns true if the predicate is satisfied by the given span. -func (pred *predicateData) satisfiedBy(span *common.Span) bool { - intVal, strVal := pred.extractRelevantSpanData(span) - if pred.fieldIsNumeric() { - switch pred.Op { - case common.EQUALS: - return intVal == pred.uintKey - case common.LESS_THAN_OR_EQUALS: - return intVal <= pred.uintKey - case common.GREATER_THAN_OR_EQUALS: - return intVal >= pred.uintKey - case common.GREATER_THAN: - return intVal > pred.uintKey - default: - panic(fmt.Sprintf("unknown Op type %s should have been caught "+ - "during normalization", pred.Op)) - } - } else { - switch pred.Op { - case common.CONTAINS: - return strings.Contains(strVal, pred.strKey) - case common.EQUALS: - return strVal == pred.strKey - case common.LESS_THAN_OR_EQUALS: - return strVal <= pred.strKey - case common.GREATER_THAN_OR_EQUALS: - return strVal >= pred.strKey - case common.GREATER_THAN: - return strVal > pred.strKey - default: - panic(fmt.Sprintf("unknown Op type %s should have been caught "+ - "during normalization", pred.Op)) - } - } -} - -func (pred *predicateData) createSource(store *dataStore, prev *common.Span) (*source, error) { - var ret *source - src := source{store: store, - pred: pred, - iters: make([]*levigo.Iterator, 0, len(store.shards)), - nexts: make([]*common.Span, len(store.shards)), - numRead: make([]int, len(store.shards)), - keyPrefix: pred.getIndexPrefix(), - } - if src.keyPrefix == INVALID_INDEX_PREFIX { - return nil, errors.New(fmt.Sprintf("Can't create source from unindexed "+ - "predicate on field %s", pred.Field)) - } - defer func() { - if ret == nil { - src.Close() - } - }() - for shardIdx := range store.shards { - shd := store.shards[shardIdx] - src.iters = append(src.iters, shd.ldb.NewIterator(store.readOpts)) - } - var searchKey []byte - lg := store.lg - if prev != nil { - // If prev != nil, this query RPC is the continuation of a previous - // one. The final result returned the last time is 'prev'. - // - // To avoid returning the same results multiple times, we adjust the - // predicate here. If the predicate is on the span id field, we - // simply manipulate the span ID we're looking for. - // - // If the predicate is on a secondary index, we also use span ID, but - // in a slightly different way. Since the secondary indices are - // organized as [type-code][8b-secondary-key][8b-span-id], elements - // with the same secondary index field are ordered by span ID. So we - // create a 17-byte key incorporating the span ID from 'prev.' - var startId common.SpanId - switch pred.Op { - case common.EQUALS: - if pred.Field == common.SPAN_ID { - // This is an annoying corner case. There can only be one - // result each time we do an EQUALS search for a span id. - // Span id is the primary key for all our spans. - // But for some reason someone is asking for another result. - // We modify the query to search for the illegal 0 span ID, - // which will never be present. - lg.Debugf("Attempted to use a continuation token with an EQUALS "+ - "SPAN_ID query. %s. Setting search id = 0", - pred.Predicate.String()) - startId = 0 - } else { - // When doing an EQUALS search on a secondary index, the - // results are sorted by span id. - startId = prev.Id + 1 - } - case common.LESS_THAN_OR_EQUALS: - // Subtract one from the previous span id. Since the previous - // start ID will never be 0 (0 is an illegal span id), we'll never - // wrap around when doing this. - startId = prev.Id - 1 - case common.GREATER_THAN_OR_EQUALS: - // We can't add one to the span id, since the previous span ID - // might be the maximum value. So just switch over to using - // GREATER_THAN. - pred.Op = common.GREATER_THAN - startId = prev.Id - case common.GREATER_THAN: - // This one is easy. - startId = prev.Id - default: - str := fmt.Sprintf("Can't use a %v predicate as a source.", pred.Predicate.String()) - lg.Error(str + "\n") - panic(str) - } - if pred.Field == common.SPAN_ID { - pred.uintKey = uint64(startId) - searchKey = makeKey(src.keyPrefix, uint64(startId)) - } else { - // Start where the previous query left off. This means adjusting - // our uintKey. - pred.uintKey, _ = pred.extractRelevantSpanData(prev) - searchKey = makeSecondaryKey(src.keyPrefix, pred.uintKey, uint64(startId)) - } - if lg.TraceEnabled() { - lg.Tracef("Handling continuation token %s for %s. startId=%d, "+ - "pred.uintKey=%d\n", prev, pred.Predicate.String(), startId, - pred.uintKey) - } - } else { - searchKey = makeKey(src.keyPrefix, pred.uintKey) - } - for i := range src.iters { - src.iters[i].Seek(searchKey) - } - ret = &src - return ret, nil -} - -// A source of spans. -type source struct { - store *dataStore - pred *predicateData - iters []*levigo.Iterator - nexts []*common.Span - numRead []int - keyPrefix byte -} - -// Return true if this operation may require skipping the first result we get back from leveldb. -func mayRequireOneSkip(op common.Op) bool { - switch op { - // When dealing with descending predicates, the first span we read might not satisfy - // the predicate, even though subsequent ones will. This is because the iter.Seek() - // function "moves the iterator the position of the key given or, if the key doesn't - // exist, the next key that does exist in the database." So if we're on that "next - // key" it will not satisfy the predicate, but the keys previous to it might. - case common.LESS_THAN_OR_EQUALS: - return true - // iter.Seek basically takes us to the key which is "greater than or equal to" some - // value. Since we want greater than (not greater than or equal to) we may have to - // skip the first key. - case common.GREATER_THAN: - return true - } - return false -} - -// Fill in the entry in the 'next' array for a specific shard. -func (src *source) populateNextFromShard(shardIdx int) { - lg := src.store.lg - var err error - iter := src.iters[shardIdx] - if iter == nil { - lg.Debugf("Can't populate: No more entries in shard %d\n", shardIdx) - return // There are no more entries in this shard. - } - if src.nexts[shardIdx] != nil { - lg.Debugf("No need to populate shard %d\n", shardIdx) - return // We already have a valid entry for this shard. - } - for { - if !iter.Valid() { - lg.Debugf("Can't populate: Iterator for shard %d is no longer valid.\n", shardIdx) - break // Can't read past end of DB - } - src.numRead[shardIdx]++ - key := iter.Key() - if !bytes.HasPrefix(key, []byte{src.keyPrefix}) { - lg.Debugf("Can't populate: Iterator for shard %d does not have prefix %s\n", - shardIdx, string(src.keyPrefix)) - break // Can't read past end of indexed section - } - var span *common.Span - var sid common.SpanId - if src.keyPrefix == SPAN_ID_INDEX_PREFIX { - // The span id maps to the span itself. - sid = common.SpanId(keyToInt(key[1:])) - span, err = src.store.shards[shardIdx].decodeSpan(sid, iter.Value()) - if err != nil { - lg.Debugf("Internal error decoding span %s in shard %d: %s\n", - sid.String(), shardIdx, err.Error()) - break - } - } else { - // With a secondary index, we have to look up the span by id. - sid = common.SpanId(keyToInt(key[9:])) - span = src.store.shards[shardIdx].FindSpan(sid) - if span == nil { - lg.Debugf("Internal error rehydrating span %s in shard %d\n", - sid.String(), shardIdx) - break - } - } - if src.pred.Op.IsDescending() { - iter.Prev() - } else { - iter.Next() - } - if src.pred.satisfiedBy(span) { - lg.Debugf("Populated valid span %v from shard %d.\n", sid, shardIdx) - src.nexts[shardIdx] = span // Found valid entry - return - } else { - lg.Debugf("Span %s from shard %d does not satisfy the predicate.\n", - sid.String(), shardIdx) - if src.numRead[shardIdx] <= 1 && mayRequireOneSkip(src.pred.Op) { - continue - } - // This and subsequent entries don't satisfy predicate - break - } - } - lg.Debugf("Closing iterator for shard %d.\n", shardIdx) - iter.Close() - src.iters[shardIdx] = nil -} - -func (src *source) next() *common.Span { - for shardIdx := range src.iters { - src.populateNextFromShard(shardIdx) - } - var best *common.Span - bestIdx := -1 - for shardIdx := range src.iters { - span := src.nexts[shardIdx] - if src.pred.spanPtrIsBefore(span, best) { - best = span - bestIdx = shardIdx - } - } - if bestIdx >= 0 { - src.nexts[bestIdx] = nil - } - return best -} - -func (src *source) Close() { - for i := range src.iters { - if src.iters[i] != nil { - src.iters[i].Close() - } - } - src.iters = nil -} - -func (store *dataStore) obtainSource(preds *[]*predicateData, span *common.Span) (*source, error) { - // Read spans from the first predicate that is indexed. - p := *preds - for i := range p { - pred := p[i] - if pred.getIndexPrefix() != INVALID_INDEX_PREFIX { - *preds = append(p[0:i], p[i+1:]...) - return pred.createSource(store, span) - } - } - // If there are no predicates that are indexed, read rows in order of span id. - spanIdPred := common.Predicate{Op: common.GREATER_THAN_OR_EQUALS, - Field: common.SPAN_ID, - Val: "0000000000000000", - } - spanIdPredData, err := loadPredicateData(&spanIdPred) - if err != nil { - return nil, err - } - return spanIdPredData.createSource(store, span) -} - -func (store *dataStore) HandleQuery(query *common.Query) ([]*common.Span, error) { - lg := store.lg - // Parse predicate data. - var err error - preds := make([]*predicateData, len(query.Predicates)) - for i := range query.Predicates { - preds[i], err = loadPredicateData(&query.Predicates[i]) - if err != nil { - return nil, err - } - } - // Get a source of rows. - var src *source - src, err = store.obtainSource(&preds, query.Prev) - if err != nil { - return nil, err - } - defer src.Close() - lg.Debugf("HandleQuery %s: preds = %s, src = %v\n", query, preds, src) - - // Filter the spans through the remaining predicates. - ret := make([]*common.Span, 0, 32) - for { - if len(ret) >= query.Lim { - break // we hit the result size limit - } - span := src.next() - if span == nil { - break // the source has no more spans to give - } - if lg.DebugEnabled() { - lg.Debugf("src.next returned span %s\n", span.ToJson()) - } - satisfied := true - for predIdx := range preds { - if !preds[predIdx].satisfiedBy(span) { - satisfied = false - break - } - } - if satisfied { - ret = append(ret, span) - } - } - return ret, nil -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htraced/datastore_test.go b/htrace-htraced/src/go/src/org/apache/htrace/htraced/datastore_test.go deleted file mode 100644 index 4696547..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htraced/datastore_test.go +++ /dev/null @@ -1,514 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bytes" - "encoding/json" - "math/rand" - htrace "org/apache/htrace/client" - "org/apache/htrace/common" - "org/apache/htrace/conf" - "org/apache/htrace/test" - "os" - "sort" - "strings" - "testing" -) - -// Test creating and tearing down a datastore. -func TestCreateDatastore(t *testing.T) { - htraceBld := &MiniHTracedBuilder{Name: "TestCreateDatastore", - DataDirs: make([]string, 3)} - ht, err := htraceBld.Build() - if err != nil { - t.Fatalf("failed to create datastore: %s", err.Error()) - } - defer ht.Close() -} - -var SIMPLE_TEST_SPANS []common.Span = []common.Span{ - common.Span{Id: 1, - SpanData: common.SpanData{ - Begin: 123, - End: 456, - Description: "getFileDescriptors", - TraceId: 999, - Parents: []common.SpanId{}, - ProcessId: "firstd", - }}, - common.Span{Id: 2, - SpanData: common.SpanData{ - Begin: 125, - End: 200, - Description: "openFd", - TraceId: 999, - Parents: []common.SpanId{1}, - ProcessId: "secondd", - }}, - common.Span{Id: 3, - SpanData: common.SpanData{ - Begin: 200, - End: 456, - Description: "passFd", - TraceId: 999, - Parents: []common.SpanId{1}, - ProcessId: "thirdd", - }}, -} - -func createSpans(spans []common.Span, store *dataStore) { - for idx := range spans { - store.WriteSpan(&spans[idx]) - } - // Wait the spans to be created - for i := 0; i < 3; i++ { - <-store.WrittenSpans - } -} - -// Test creating a datastore and adding some spans. -func TestDatastoreWriteAndRead(t *testing.T) { - t.Parallel() - htraceBld := &MiniHTracedBuilder{Name: "TestDatastoreWriteAndRead", - WrittenSpans: make(chan *common.Span, 100)} - ht, err := htraceBld.Build() - if err != nil { - panic(err) - } - defer ht.Close() - createSpans(SIMPLE_TEST_SPANS, ht.Store) - if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { - t.Fatal() - } - span := ht.Store.FindSpan(1) - if span == nil { - t.Fatal() - } - if span.Id != 1 { - t.Fatal() - } - common.ExpectSpansEqual(t, &SIMPLE_TEST_SPANS[0], span) - children := ht.Store.FindChildren(1, 1) - if len(children) != 1 { - t.Fatalf("expected 1 child, but got %d\n", len(children)) - } - children = ht.Store.FindChildren(1, 2) - if len(children) != 2 { - t.Fatalf("expected 2 children, but got %d\n", len(children)) - } - sort.Sort(common.SpanIdSlice(children)) - if children[0] != 2 { - t.Fatal() - } - if children[1] != 3 { - t.Fatal() - } -} - -func testQuery(t *testing.T, ht *MiniHTraced, query *common.Query, - expectedSpans []common.Span) { - spans, err := ht.Store.HandleQuery(query) - if err != nil { - t.Fatalf("First query failed: %s\n", err.Error()) - } - expectedBuf := new(bytes.Buffer) - dec := json.NewEncoder(expectedBuf) - err = dec.Encode(expectedSpans) - if err != nil { - t.Fatalf("Failed to encode expectedSpans to JSON: %s\n", err.Error()) - } - spansBuf := new(bytes.Buffer) - dec = json.NewEncoder(spansBuf) - err = dec.Encode(spans) - if err != nil { - t.Fatalf("Failed to encode result spans to JSON: %s\n", err.Error()) - } - t.Logf("len(spans) = %d, len(expectedSpans) = %d\n", len(spans), - len(expectedSpans)) - common.ExpectStrEqual(t, string(expectedBuf.Bytes()), string(spansBuf.Bytes())) -} - -// Test queries on the datastore. -func TestSimpleQuery(t *testing.T) { - t.Parallel() - htraceBld := &MiniHTracedBuilder{Name: "TestSimpleQuery", - WrittenSpans: make(chan *common.Span, 100)} - ht, err := htraceBld.Build() - if err != nil { - panic(err) - } - defer ht.Close() - createSpans(SIMPLE_TEST_SPANS, ht.Store) - if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { - t.Fatal() - } - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.GREATER_THAN_OR_EQUALS, - Field: common.BEGIN_TIME, - Val: "125", - }, - }, - Lim: 5, - }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[2]}) -} - -func TestQueries2(t *testing.T) { - t.Parallel() - htraceBld := &MiniHTracedBuilder{Name: "TestQueries2", - WrittenSpans: make(chan *common.Span, 100)} - ht, err := htraceBld.Build() - if err != nil { - panic(err) - } - defer ht.Close() - createSpans(SIMPLE_TEST_SPANS, ht.Store) - if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { - t.Fatal() - } - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.LESS_THAN_OR_EQUALS, - Field: common.BEGIN_TIME, - Val: "125", - }, - }, - Lim: 5, - }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[0]}) - - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.LESS_THAN_OR_EQUALS, - Field: common.BEGIN_TIME, - Val: "125", - }, - common.Predicate{ - Op: common.EQUALS, - Field: common.DESCRIPTION, - Val: "getFileDescriptors", - }, - }, - Lim: 2, - }, []common.Span{SIMPLE_TEST_SPANS[0]}) - - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.EQUALS, - Field: common.DESCRIPTION, - Val: "getFileDescriptors", - }, - }, - Lim: 2, - }, []common.Span{SIMPLE_TEST_SPANS[0]}) -} - -func TestQueries3(t *testing.T) { - t.Parallel() - htraceBld := &MiniHTracedBuilder{Name: "TestQueries3", - WrittenSpans: make(chan *common.Span, 100)} - ht, err := htraceBld.Build() - if err != nil { - panic(err) - } - defer ht.Close() - createSpans(SIMPLE_TEST_SPANS, ht.Store) - if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { - t.Fatal() - } - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.CONTAINS, - Field: common.DESCRIPTION, - Val: "Fd", - }, - common.Predicate{ - Op: common.GREATER_THAN_OR_EQUALS, - Field: common.BEGIN_TIME, - Val: "100", - }, - }, - Lim: 5, - }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[2]}) - - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.LESS_THAN_OR_EQUALS, - Field: common.SPAN_ID, - Val: "0", - }, - }, - Lim: 200, - }, []common.Span{}) - - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.LESS_THAN_OR_EQUALS, - Field: common.SPAN_ID, - Val: "2", - }, - }, - Lim: 200, - }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[0]}) -} - -func TestQueries4(t *testing.T) { - t.Parallel() - htraceBld := &MiniHTracedBuilder{Name: "TestQueries4", - WrittenSpans: make(chan *common.Span, 100)} - ht, err := htraceBld.Build() - if err != nil { - panic(err) - } - defer ht.Close() - createSpans(SIMPLE_TEST_SPANS, ht.Store) - if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { - t.Fatal() - } - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.GREATER_THAN, - Field: common.BEGIN_TIME, - Val: "125", - }, - }, - Lim: 5, - }, []common.Span{SIMPLE_TEST_SPANS[2]}) - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.GREATER_THAN_OR_EQUALS, - Field: common.DESCRIPTION, - Val: "openFd", - }, - }, - Lim: 2, - }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[2]}) - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.GREATER_THAN, - Field: common.DESCRIPTION, - Val: "openFd", - }, - }, - Lim: 2, - }, []common.Span{SIMPLE_TEST_SPANS[2]}) -} - -func BenchmarkDatastoreWrites(b *testing.B) { - htraceBld := &MiniHTracedBuilder{Name: "BenchmarkDatastoreWrites", - WrittenSpans: make(chan *common.Span, b.N)} - ht, err := htraceBld.Build() - if err != nil { - panic(err) - } - defer ht.Close() - rnd := rand.New(rand.NewSource(1)) - allSpans := make([]*common.Span, b.N) - // Write many random spans. - for n := 0; n < b.N; n++ { - span := test.NewRandomSpan(rnd, allSpans[0:n]) - ht.Store.WriteSpan(span) - allSpans[n] = span - } - // Wait for all the spans to be written. - for n := 0; n < b.N; n++ { - <-ht.Store.WrittenSpans - } - spansWritten := ht.Store.GetStatistics().NumSpansWritten - if spansWritten < uint64(b.N) { - b.Fatal("incorrect statistics: expected %d spans to be written, but only got %d", - b.N, spansWritten) - } -} - -func TestReloadDataStore(t *testing.T) { - htraceBld := &MiniHTracedBuilder{Name: "TestReloadDataStore", - DataDirs: make([]string, 2), KeepDataDirsOnClose: true} - ht, err := htraceBld.Build() - if err != nil { - t.Fatalf("failed to create datastore: %s", err.Error()) - } - dataDirs := make([]string, len(ht.DataDirs)) - copy(dataDirs, ht.DataDirs) - defer func() { - if ht != nil { - ht.Close() - } - for i := range dataDirs { - os.RemoveAll(dataDirs[i]) - } - }() - var hcl *htrace.Client - hcl, err = htrace.NewClient(ht.ClientConf()) - if err != nil { - t.Fatalf("failed to create client: %s", err.Error()) - } - - // Create some random trace spans. - NUM_TEST_SPANS := 5 - allSpans := createRandomTestSpans(NUM_TEST_SPANS) - err = hcl.WriteSpans(&common.WriteSpansReq{ - Spans: allSpans, - }) - if err != nil { - t.Fatalf("WriteSpans failed: %s\n", err.Error()) - } - - // Look up the spans we wrote. - var span *common.Span - for i := 0; i < NUM_TEST_SPANS; i++ { - span, err = hcl.FindSpan(allSpans[i].Id) - if err != nil { - t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) - } - common.ExpectSpansEqual(t, allSpans[i], span) - } - - ht.Close() - ht = nil - - htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore2", - DataDirs: dataDirs, KeepDataDirsOnClose: true} - ht, err = htraceBld.Build() - if err != nil { - t.Fatalf("failed to re-create datastore: %s", err.Error()) - } - hcl, err = htrace.NewClient(ht.ClientConf()) - if err != nil { - t.Fatalf("failed to re-create client: %s", err.Error()) - } - - // Look up the spans we wrote earlier. - for i := 0; i < NUM_TEST_SPANS; i++ { - span, err = hcl.FindSpan(allSpans[i].Id) - if err != nil { - t.Fatalf("FindSpan(%d) failed: %s\n", i, err.Error()) - } - common.ExpectSpansEqual(t, allSpans[i], span) - } - - // Set an old datastore version number. - for i := range ht.Store.shards { - shard := ht.Store.shards[i] - writeDataStoreVersion(ht.Store, shard.ldb, CURRENT_LAYOUT_VERSION-1) - } - ht.Close() - ht = nil - - htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore3", - DataDirs: dataDirs, KeepDataDirsOnClose: true} - ht, err = htraceBld.Build() - if err == nil { - t.Fatalf("expected the datastore to fail to load after setting an " + - "incorrect version.\n") - } - if !strings.Contains(err.Error(), "Invalid layout version") { - t.Fatal(`expected the loading error to contain "invalid layout version"` + "\n") - } - - // It should work with data.store.clear set. - htraceBld = &MiniHTracedBuilder{Name: "TestReloadDataStore4", - DataDirs: dataDirs, KeepDataDirsOnClose: true, - Cnf: map[string]string{conf.HTRACE_DATA_STORE_CLEAR: "true"}} - ht, err = htraceBld.Build() - if err != nil { - t.Fatalf("expected the datastore loading to succeed after setting an "+ - "incorrect version. But it failed with error %s\n", err.Error()) - } -} - -func TestQueriesWithContinuationTokens1(t *testing.T) { - t.Parallel() - htraceBld := &MiniHTracedBuilder{Name: "TestQueriesWithContinuationTokens1", - WrittenSpans: make(chan *common.Span, 100)} - ht, err := htraceBld.Build() - if err != nil { - panic(err) - } - defer ht.Close() - createSpans(SIMPLE_TEST_SPANS, ht.Store) - if ht.Store.GetStatistics().NumSpansWritten < uint64(len(SIMPLE_TEST_SPANS)) { - t.Fatal() - } - // Adding a prev value to this query excludes the first result that we - // would normally get. - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.GREATER_THAN, - Field: common.BEGIN_TIME, - Val: "120", - }, - }, - Lim: 5, - Prev: &SIMPLE_TEST_SPANS[0], - }, []common.Span{SIMPLE_TEST_SPANS[1], SIMPLE_TEST_SPANS[2]}) - - // There is only one result from an EQUALS query on SPAN_ID. - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.EQUALS, - Field: common.SPAN_ID, - Val: "1", - }, - }, - Lim: 100, - Prev: &SIMPLE_TEST_SPANS[0], - }, []common.Span{}) - - // When doing a LESS_THAN_OR_EQUALS search, we still don't get back the - // span we pass as a continuation token. (Primary index edition). - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.LESS_THAN_OR_EQUALS, - Field: common.SPAN_ID, - Val: "2", - }, - }, - Lim: 100, - Prev: &SIMPLE_TEST_SPANS[1], - }, []common.Span{SIMPLE_TEST_SPANS[0]}) - - // When doing a GREATER_THAN_OR_EQUALS search, we still don't get back the - // span we pass as a continuation token. (Secondary index edition). - testQuery(t, ht, &common.Query{ - Predicates: []common.Predicate{ - common.Predicate{ - Op: common.GREATER_THAN, - Field: common.DURATION, - Val: "0", - }, - }, - Lim: 100, - Prev: &SIMPLE_TEST_SPANS[1], - }, []common.Span{SIMPLE_TEST_SPANS[2], SIMPLE_TEST_SPANS[0]}) -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htraced/hrpc.go b/htrace-htraced/src/go/src/org/apache/htrace/htraced/hrpc.go deleted file mode 100644 index a53380e..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htraced/hrpc.go +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bufio" - "bytes" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "github.com/ugorji/go/codec" - "io" - "net" - "net/rpc" - "org/apache/htrace/common" - "org/apache/htrace/conf" -) - -// Handles HRPC calls -type HrpcHandler struct { - lg *common.Logger - store *dataStore -} - -// The HRPC server -type HrpcServer struct { - *rpc.Server - hand *HrpcHandler - listener net.Listener -} - -// Codec which encodes HRPC data via JSON -type HrpcServerCodec struct { - lg *common.Logger - conn net.Conn - length uint32 -} - -func asJson(val interface{}) string { - js, err := json.Marshal(val) - if err != nil { - return "encoding error: " + err.Error() - } - return string(js) -} - -func createErrAndWarn(lg *common.Logger, val string) error { - return createErrAndLog(lg, val, common.WARN) -} - -func createErrAndLog(lg *common.Logger, val string, level common.Level) error { - lg.Write(level, val+"\n") - return errors.New(val) -} - -func (cdc *HrpcServerCodec) ReadRequestHeader(req *rpc.Request) error { - hdr := common.HrpcRequestHeader{} - if cdc.lg.TraceEnabled() { - cdc.lg.Tracef("Reading HRPC request header from %s\n", cdc.conn.RemoteAddr()) - } - err := binary.Read(cdc.conn, binary.LittleEndian, &hdr) - if err != nil { - level := common.WARN - if err == io.EOF { - level = common.DEBUG - } - return createErrAndLog(cdc.lg, fmt.Sprintf("Error reading header bytes: %s", - err.Error()), level) - } - if cdc.lg.TraceEnabled() { - cdc.lg.Tracef("Read HRPC request header %s from %s\n", - asJson(&hdr), cdc.conn.RemoteAddr()) - } - if hdr.Magic != common.HRPC_MAGIC { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Invalid request header: expected "+ - "magic number of 0x%04x, but got 0x%04x", common.HRPC_MAGIC, hdr.Magic)) - } - if hdr.Length > common.MAX_HRPC_BODY_LENGTH { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Length prefix was too long. "+ - "Maximum length is %d, but we got %d.", common.MAX_HRPC_BODY_LENGTH, - hdr.Length)) - } - req.ServiceMethod = common.HrpcMethodIdToMethodName(hdr.MethodId) - if req.ServiceMethod == "" { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Unknown MethodID code 0x%04x", - hdr.MethodId)) - } - req.Seq = hdr.Seq - cdc.length = hdr.Length - return nil -} - -func (cdc *HrpcServerCodec) ReadRequestBody(body interface{}) error { - if cdc.lg.TraceEnabled() { - cdc.lg.Tracef("Reading HRPC %d-byte request body from %s\n", - cdc.length, cdc.conn.RemoteAddr()) - } - mh := new(codec.MsgpackHandle) - mh.WriteExt = true - dec := codec.NewDecoder(io.LimitReader(cdc.conn, int64(cdc.length)), mh) - err := dec.Decode(body) - if err != nil { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to read request "+ - "body from %s: %s", cdc.conn.RemoteAddr(), err.Error())) - } - if cdc.lg.TraceEnabled() { - cdc.lg.Tracef("Read body from %s: %s\n", - cdc.conn.RemoteAddr(), asJson(&body)) - } - return nil -} - -var EMPTY []byte = make([]byte, 0) - -func (cdc *HrpcServerCodec) WriteResponse(resp *rpc.Response, msg interface{}) error { - var err error - buf := EMPTY - if msg != nil { - mh := new(codec.MsgpackHandle) - mh.WriteExt = true - w := bytes.NewBuffer(make([]byte, 0, 128)) - enc := codec.NewEncoder(w, mh) - err := enc.Encode(msg) - if err != nil { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to marshal "+ - "response message: %s", err.Error())) - } - buf = w.Bytes() - } - hdr := common.HrpcResponseHeader{} - hdr.MethodId = common.HrpcMethodNameToId(resp.ServiceMethod) - hdr.Seq = resp.Seq - hdr.ErrLength = uint32(len(resp.Error)) - hdr.Length = uint32(len(buf)) - writer := bufio.NewWriterSize(cdc.conn, 256) - err = binary.Write(writer, binary.LittleEndian, &hdr) - if err != nil { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write response "+ - "header: %s", err.Error())) - } - if hdr.ErrLength > 0 { - _, err = io.WriteString(writer, resp.Error) - if err != nil { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write error "+ - "string: %s", err.Error())) - } - } - if hdr.Length > 0 { - var length int - length, err = writer.Write(buf) - if err != nil { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write response "+ - "message: %s", err.Error())) - } - if uint32(length) != hdr.Length { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write all of "+ - "response message: %s", err.Error())) - } - } - err = writer.Flush() - if err != nil { - return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write the response "+ - "bytes: %s", err.Error())) - } - return nil -} - -func (cdc *HrpcServerCodec) Close() error { - return cdc.conn.Close() -} - -func (hand *HrpcHandler) WriteSpans(req *common.WriteSpansReq, - resp *common.WriteSpansResp) (err error) { - hand.lg.Debugf("hrpc writeSpansHandler: received %d span(s). "+ - "defaultPid = %s\n", len(req.Spans), req.DefaultPid) - for i := range req.Spans { - span := req.Spans[i] - if span.ProcessId == "" { - span.ProcessId = req.DefaultPid - } - if hand.lg.TraceEnabled() { - hand.lg.Tracef("writing span %d: %s\n", i, span.ToJson()) - } - hand.store.WriteSpan(span) - } - return nil -} - -func CreateHrpcServer(cnf *conf.Config, store *dataStore) (*HrpcServer, error) { - lg := common.NewLogger("hrpc", cnf) - hsv := &HrpcServer{ - Server: rpc.NewServer(), - hand: &HrpcHandler{ - lg: lg, - store: store, - }, - } - var err error - hsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_HRPC_ADDRESS)) - if err != nil { - return nil, err - } - hsv.Server.Register(hsv.hand) - go hsv.run() - lg.Infof("Started HRPC server on %s...\n", hsv.listener.Addr().String()) - return hsv, nil -} - -func (hsv *HrpcServer) run() { - lg := hsv.hand.lg - for { - conn, err := hsv.listener.Accept() - if err != nil { - lg.Errorf("HRPC Accept error: %s\n", err.Error()) - continue - } - if lg.TraceEnabled() { - lg.Tracef("Accepted HRPC connection from %s\n", conn.RemoteAddr()) - } - go hsv.ServeCodec(&HrpcServerCodec{ - lg: lg, - conn: conn, - }) - } -} - -func (hsv *HrpcServer) Addr() net.Addr { - return hsv.listener.Addr() -} - -func (hsv *HrpcServer) Close() { - hsv.listener.Close() -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htraced/htraced.go b/htrace-htraced/src/go/src/org/apache/htrace/htraced/htraced.go deleted file mode 100644 index 64da457..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htraced/htraced.go +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "encoding/json" - "fmt" - "net" - "org/apache/htrace/common" - "org/apache/htrace/conf" - "os" - "strings" - "time" -) - -var RELEASE_VERSION string -var GIT_VERSION string - -const USAGE = `htraced: the HTrace server daemon. - -htraced receives trace spans sent from HTrace clients. It exposes a REST -interface which others can query. It also runs a web server with a graphical -user interface. htraced stores its span data in levelDB files on the local -disks. - -Usage: ---help: this help message - --Dk=v: set configuration key 'k' to value 'v' -For example -Dweb.address=127.0.0.1:8080 sets the web address to localhost, -port 8080. - --Dk: set configuration key 'k' to 'true' - -Normally, configuration options should be set in the ` + conf.CONFIG_FILE_NAME + ` -configuration file. We find this file by searching the paths in the -` + conf.HTRACED_CONF_DIR + `. The command-line options are just an alternate way -of setting configuration when launching the daemon. -` - -func main() { - for idx := range os.Args { - arg := os.Args[idx] - if strings.HasPrefix(arg, "--h") || strings.HasPrefix(arg, "-h") { - fmt.Fprintf(os.Stderr, USAGE) - os.Exit(0) - } - } - cnf := common.LoadApplicationConfig() - common.InstallSignalHandlers(cnf) - lg := common.NewLogger("main", cnf) - defer lg.Close() - store, err := CreateDataStore(cnf, nil) - if err != nil { - lg.Errorf("Error creating datastore: %s\n", err.Error()) - os.Exit(1) - } - var rsv *RestServer - rsv, err = CreateRestServer(cnf, store) - if err != nil { - lg.Errorf("Error creating REST server: %s\n", err.Error()) - os.Exit(1) - } - var hsv *HrpcServer - if cnf.Get(conf.HTRACE_HRPC_ADDRESS) != "" { - hsv, err = CreateHrpcServer(cnf, store) - if err != nil { - lg.Errorf("Error creating HRPC server: %s\n", err.Error()) - os.Exit(1) - } - } else { - lg.Infof("Not starting HRPC server because no value was given for %s.\n", - conf.HTRACE_HRPC_ADDRESS) - } - naddr := cnf.Get(conf.HTRACE_STARTUP_NOTIFICATION_ADDRESS) - if naddr != "" { - notif := StartupNotification{ - HttpAddr: rsv.Addr().String(), - ProcessId: os.Getpid(), - } - if hsv != nil { - notif.HrpcAddr = hsv.Addr().String() - } - err = sendStartupNotification(naddr, ¬if) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to send startup notification: "+ - "%s\n", err.Error()) - os.Exit(1) - } - } - for { - time.Sleep(time.Duration(10) * time.Hour) - } -} - -// A startup notification message that we optionally send on startup. -// Used by unit tests. -type StartupNotification struct { - HttpAddr string - HrpcAddr string - ProcessId int -} - -func sendStartupNotification(naddr string, notif *StartupNotification) error { - conn, err := net.Dial("tcp", naddr) - if err != nil { - return err - } - defer func() { - if conn != nil { - conn.Close() - } - }() - var buf []byte - buf, err = json.Marshal(notif) - if err != nil { - return err - } - _, err = conn.Write(buf) - conn.Close() - conn = nil - return nil -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htraced/mini_htraced.go b/htrace-htraced/src/go/src/org/apache/htrace/htraced/mini_htraced.go deleted file mode 100644 index a54f2cb..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htraced/mini_htraced.go +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "fmt" - "io/ioutil" - "org/apache/htrace/common" - "org/apache/htrace/conf" - "os" - "strings" -) - -// -// MiniHTraceD is used in unit tests to set up a daemon with certain settings. -// It takes care of things like creating and cleaning up temporary directories. -// - -// The default number of managed data directories to use. -const DEFAULT_NUM_DATA_DIRS = 2 - -// Builds a MiniHTraced object. -type MiniHTracedBuilder struct { - // The name of the MiniHTraced to build. This shows up in the test directory name and some - // other places. - Name string - - // The configuration values to use for the MiniHTraced. - // If ths is nil, we use the default configuration for everything. - Cnf map[string]string - - // The DataDirs to use. Empty entries will turn into random names. - DataDirs []string - - // If true, we will keep the data dirs around after MiniHTraced#Close - KeepDataDirsOnClose bool - - // If non-null, the WrittenSpans channel to use when creating the DataStore. - WrittenSpans chan *common.Span -} - -type MiniHTraced struct { - Name string - Cnf *conf.Config - DataDirs []string - Store *dataStore - Rsv *RestServer - Hsv *HrpcServer - Lg *common.Logger - KeepDataDirsOnClose bool -} - -func (bld *MiniHTracedBuilder) Build() (*MiniHTraced, error) { - var err error - var store *dataStore - var rsv *RestServer - var hsv *HrpcServer - if bld.Name == "" { - bld.Name = "HTraceTest" - } - if bld.Cnf == nil { - bld.Cnf = make(map[string]string) - } - if bld.DataDirs == nil { - bld.DataDirs = make([]string, 2) - } - for idx := range bld.DataDirs { - if bld.DataDirs[idx] == "" { - bld.DataDirs[idx], err = ioutil.TempDir(os.TempDir(), - fmt.Sprintf("%s%d", bld.Name, idx+1)) - if err != nil { - return nil, err - } - } - } - bld.Cnf[conf.HTRACE_DATA_STORE_DIRECTORIES] = - strings.Join(bld.DataDirs, conf.PATH_LIST_SEP) - bld.Cnf[conf.HTRACE_WEB_ADDRESS] = ":0" // use a random port for the REST server - bld.Cnf[conf.HTRACE_HRPC_ADDRESS] = ":0" // use a random port for the HRPC server - bld.Cnf[conf.HTRACE_LOG_LEVEL] = "TRACE" - cnfBld := conf.Builder{Values: bld.Cnf, Defaults: conf.DEFAULTS} - cnf, err := cnfBld.Build() - if err != nil { - return nil, err - } - lg := common.NewLogger("mini.htraced", cnf) - defer func() { - if err != nil { - if store != nil { - store.Close() - } - for idx := range bld.DataDirs { - if bld.DataDirs[idx] != "" { - os.RemoveAll(bld.DataDirs[idx]) - } - } - if rsv != nil { - rsv.Close() - } - lg.Infof("Failed to create MiniHTraced %s: %s\n", bld.Name, err.Error()) - lg.Close() - } - }() - store, err = CreateDataStore(cnf, bld.WrittenSpans) - if err != nil { - return nil, err - } - rsv, err = CreateRestServer(cnf, store) - if err != nil { - return nil, err - } - hsv, err = CreateHrpcServer(cnf, store) - if err != nil { - return nil, err - } - - lg.Infof("Created MiniHTraced %s\n", bld.Name) - return &MiniHTraced{ - Name: bld.Name, - Cnf: cnf, - DataDirs: bld.DataDirs, - Store: store, - Rsv: rsv, - Hsv: hsv, - Lg: lg, - KeepDataDirsOnClose: bld.KeepDataDirsOnClose, - }, nil -} - -// Return a Config object that clients can use to connect to this MiniHTraceD. -func (ht *MiniHTraced) ClientConf() *conf.Config { - return ht.Cnf.Clone(conf.HTRACE_WEB_ADDRESS, ht.Rsv.Addr().String(), - conf.HTRACE_HRPC_ADDRESS, ht.Hsv.Addr().String()) -} - -func (ht *MiniHTraced) Close() { - ht.Lg.Infof("Closing MiniHTraced %s\n", ht.Name) - ht.Rsv.Close() - ht.Store.Close() - if !ht.KeepDataDirsOnClose { - for idx := range ht.DataDirs { - ht.Lg.Infof("Removing %s...\n", ht.DataDirs[idx]) - os.RemoveAll(ht.DataDirs[idx]) - } - } - ht.Lg.Infof("Finished closing MiniHTraced %s\n", ht.Name) - ht.Lg.Close() -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/htraced/rest.go b/htrace-htraced/src/go/src/org/apache/htrace/htraced/rest.go deleted file mode 100644 index 69b316c..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/htraced/rest.go +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "github.com/gorilla/mux" - "io" - "net" - "net/http" - "org/apache/htrace/common" - "org/apache/htrace/conf" - "os" - "path/filepath" - "strconv" - "strings" -) - -// Set the response headers. -func setResponseHeaders(hdr http.Header) { - hdr.Set("Content-Type", "application/json") -} - -// Write a JSON error response. -func writeError(lg *common.Logger, w http.ResponseWriter, errCode int, - errStr string) { - str := strings.Replace(errStr, `"`, `'`, -1) - lg.Info(str + "\n") - w.WriteHeader(errCode) - w.Write([]byte(`{ "error" : "` + str + `"}`)) -} - -type serverInfoHandler struct { - lg *common.Logger -} - -func (hand *serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - setResponseHeaders(w.Header()) - version := common.ServerInfo{ReleaseVersion: RELEASE_VERSION, - GitVersion: GIT_VERSION} - buf, err := json.Marshal(&version) - if err != nil { - writeError(hand.lg, w, http.StatusInternalServerError, - fmt.Sprintf("error marshalling ServerInfo: %s\n", err.Error())) - return - } - if hand.lg.DebugEnabled() { - hand.lg.Debugf("Returned serverInfo %s\n", string(buf)) - } - w.Write(buf) -} - -type dataStoreHandler struct { - lg *common.Logger - store *dataStore -} - -func (hand *dataStoreHandler) parseSid(w http.ResponseWriter, - str string) (common.SpanId, bool) { - val, err := strconv.ParseUint(str, 16, 64) - if err != nil { - writeError(hand.lg, w, http.StatusBadRequest, - fmt.Sprintf("Failed to parse span ID %s: %s", str, err.Error())) - w.Write([]byte("Error parsing : " + err.Error())) - return 0, false - } - return common.SpanId(val), true -} - -func (hand *dataStoreHandler) getReqField32(fieldName string, w http.ResponseWriter, - req *http.Request) (int32, bool) { - str := req.FormValue(fieldName) - if str == "" { - writeError(hand.lg, w, http.StatusBadRequest, fmt.Sprintf("No %s specified.", fieldName)) - return -1, false - } - val, err := strconv.ParseUint(str, 16, 32) - if err != nil { - writeError(hand.lg, w, http.StatusBadRequest, - fmt.Sprintf("Error parsing %s: %s.", fieldName, err.Error())) - return -1, false - } - return int32(val), true -} - -type findSidHandler struct { - dataStoreHandler -} - -func (hand *findSidHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - setResponseHeaders(w.Header()) - req.ParseForm() - vars := mux.Vars(req) - stringSid := vars["id"] - sid, ok := hand.parseSid(w, stringSid) - if !ok { - return - } - hand.lg.Debugf("findSidHandler(sid=%s)\n", sid.String()) - span := hand.store.FindSpan(sid) - if span == nil { - writeError(hand.lg, w, http.StatusNoContent, - fmt.Sprintf("No such span as %s\n", sid.String())) - return - } - w.Write(span.ToJson()) -} - -type findChildrenHandler struct { - dataStoreHandler -} - -func (hand *findChildrenHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - setResponseHeaders(w.Header()) - req.ParseForm() - vars := mux.Vars(req) - stringSid := vars["id"] - sid, ok := hand.parseSid(w, stringSid) - if !ok { - return - } - var lim int32 - lim, ok = hand.getReqField32("lim", w, req) - if !ok { - return - } - hand.lg.Debugf("findChildrenHandler(sid=%s, lim=%d)\n", sid.String(), lim) - children := hand.store.FindChildren(sid, lim) - jbytes, err := json.Marshal(children) - if err != nil { - writeError(hand.lg, w, http.StatusInternalServerError, - fmt.Sprintf("Error marshalling children: %s", err.Error())) - return - } - w.Write(jbytes) -} - -type writeSpansHandler struct { - dataStoreHandler -} - -func (hand *writeSpansHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - setResponseHeaders(w.Header()) - dec := json.NewDecoder(req.Body) - spans := make([]*common.Span, 0, 32) - defaultPid := req.Header.Get("htrace-pid") - for { - var span common.Span - err := dec.Decode(&span) - if err != nil { - if err != io.EOF { - writeError(hand.lg, w, http.StatusBadRequest, - fmt.Sprintf("Error parsing spans: %s", err.Error())) - return - } - break - } - if span.ProcessId == "" { - span.ProcessId = defaultPid - } - spans = append(spans, &span) - } - hand.lg.Debugf("writeSpansHandler: received %d span(s). defaultPid = %s\n", - len(spans), defaultPid) - for spanIdx := range spans { - if hand.lg.DebugEnabled() { - hand.lg.Debugf("writing span %s\n", spans[spanIdx].ToJson()) - } - hand.store.WriteSpan(spans[spanIdx]) - } -} - -type queryHandler struct { - lg *common.Logger - dataStoreHandler -} - -func (hand *queryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - setResponseHeaders(w.Header()) - queryString := req.FormValue("query") - if queryString == "" { - writeError(hand.lg, w, http.StatusBadRequest, "No query provided.\n") - return - } - var query common.Query - reader := bytes.NewBufferString(queryString) - dec := json.NewDecoder(reader) - err := dec.Decode(&query) - if err != nil { - writeError(hand.lg, w, http.StatusBadRequest, - fmt.Sprintf("Error parsing query: %s", err.Error())) - return - } - var results []*common.Span - results, err = hand.store.HandleQuery(&query) - if err != nil { - writeError(hand.lg, w, http.StatusInternalServerError, - fmt.Sprintf("Internal error processing query %s: %s", - query.String(), err.Error())) - return - } - var jbytes []byte - jbytes, err = json.Marshal(results) - if err != nil { - writeError(hand.lg, w, http.StatusInternalServerError, - fmt.Sprintf("Error marshalling results: %s", err.Error())) - return - } - w.Write(jbytes) -} - -type logErrorHandler struct { - lg *common.Logger -} - -func (hand *logErrorHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - hand.lg.Errorf("Got unknown request %s\n", req.RequestURI) - writeError(hand.lg, w, http.StatusBadRequest, "Unknown request.") -} - -type RestServer struct { - listener net.Listener - lg *common.Logger -} - -func CreateRestServer(cnf *conf.Config, store *dataStore) (*RestServer, error) { - var err error - rsv := &RestServer{} - rsv.listener, err = net.Listen("tcp", cnf.Get(conf.HTRACE_WEB_ADDRESS)) - if err != nil { - return nil, err - } - var success bool - defer func() { - if !success { - rsv.Close() - } - }() - rsv.lg = common.NewLogger("rest", cnf) - - r := mux.NewRouter().StrictSlash(false) - - r.Handle("/server/info", &serverInfoHandler{lg: rsv.lg}).Methods("GET") - - writeSpansH := &writeSpansHandler{dataStoreHandler: dataStoreHandler{ - store: store, lg: rsv.lg}} - r.Handle("/writeSpans", writeSpansH).Methods("POST") - - queryH := &queryHandler{lg: rsv.lg, dataStoreHandler: dataStoreHandler{store: store}} - r.Handle("/query", queryH).Methods("GET") - - span := r.PathPrefix("/span").Subrouter() - findSidH := &findSidHandler{dataStoreHandler: dataStoreHandler{store: store, lg: rsv.lg}} - span.Handle("/{id}", findSidH).Methods("GET") - - findChildrenH := &findChildrenHandler{dataStoreHandler: dataStoreHandler{store: store, - lg: rsv.lg}} - span.Handle("/{id}/children", findChildrenH).Methods("GET") - - // Default Handler. This will serve requests for static requests. - webdir := os.Getenv("HTRACED_WEB_DIR") - if webdir == "" { - webdir, err = filepath.Abs(filepath.Join(filepath.Dir(os.Args[0]), "..", "..", "web")) - - if err != nil { - return nil, err - } - } - - rsv.lg.Infof("Serving static files from %s\n.", webdir) - r.PathPrefix("/").Handler(http.FileServer(http.Dir(webdir))).Methods("GET") - - // Log an error message for unknown non-GET requests. - r.PathPrefix("/").Handler(&logErrorHandler{lg: rsv.lg}) - - go http.Serve(rsv.listener, r) - - rsv.lg.Infof("Started REST server on %s...\n", rsv.listener.Addr().String()) - success = true - return rsv, nil -} - -func (rsv *RestServer) Addr() net.Addr { - return rsv.listener.Addr() -} - -func (rsv *RestServer) Close() { - rsv.listener.Close() -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/test/random.go b/htrace-htraced/src/go/src/org/apache/htrace/test/random.go deleted file mode 100644 index d10e2f9..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/test/random.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package test - -import ( - "fmt" - "math/rand" - "org/apache/htrace/common" -) - -func NonZeroRand64(rnd *rand.Rand) int64 { - for { - r := rnd.Int63() - if r == 0 { - continue - } - if rnd.Intn(1) != 0 { - return -r - } - return r - } -} - -func NonZeroRand32(rnd *rand.Rand) int32 { - for { - r := rnd.Int31() - if r == 0 { - continue - } - if rnd.Intn(1) != 0 { - return -r - } - return r - } -} - -// Create a random span. -func NewRandomSpan(rnd *rand.Rand, potentialParents []*common.Span) *common.Span { - parents := []common.SpanId{} - if potentialParents != nil { - parentIdx := rnd.Intn(len(potentialParents) + 1) - if parentIdx < len(potentialParents) { - parents = []common.SpanId{potentialParents[parentIdx].Id} - } - } - return &common.Span{Id: common.SpanId(NonZeroRand64(rnd)), - SpanData: common.SpanData{ - Begin: NonZeroRand64(rnd), - End: NonZeroRand64(rnd), - Description: "getFileDescriptors", - TraceId: common.SpanId(NonZeroRand64(rnd)), - Parents: parents, - ProcessId: fmt.Sprintf("process%d", NonZeroRand32(rnd)), - }} -} diff --git a/htrace-htraced/src/go/src/org/apache/htrace/test/util.go b/htrace-htraced/src/go/src/org/apache/htrace/test/util.go deleted file mode 100644 index cc058e0..0000000 --- a/htrace-htraced/src/go/src/org/apache/htrace/test/util.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package test - -import ( - "org/apache/htrace/common" -) - -func SpanId(str string) common.SpanId { - var spanId common.SpanId - err := spanId.FromString(str) - if err != nil { - panic(err.Error()) - } - return spanId -}