diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java index 863039e..bf2b5e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java @@ -24,6 +24,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.List; +import java.util.Map; import com.google.common.base.Preconditions; import org.apache.commons.cli.CommandLine; @@ -174,24 +175,22 @@ public int resolve(String [] args) { ServiceRecord record = registry.resolve(argsList.get(1)); for (Endpoint endpoint : record.external) { - if ((endpoint.protocolType.equals(ProtocolTypes.PROTOCOL_WEBUI)) - || (endpoint.protocolType.equals(ProtocolTypes.PROTOCOL_REST))) { - sysout.print(" Endpoint(ProtocolType=" - + endpoint.protocolType + ", Api=" - + endpoint.api + "); Uris are: "); - } else { - sysout.print(" Endpoint(ProtocolType=" + sysout.println(" Endpoint(ProtocolType=" + endpoint.protocolType + ", Api=" + endpoint.api + ");" + " Addresses(AddressType=" + endpoint.addressType + ") are: "); - } - for (List a : endpoint.addresses) { - sysout.print(a + " "); - } - sysout.println(); - } + for (Map address : endpoint.addresses) { + sysout.println(" [ "); + for (Map.Entry entry : address.entrySet()) { + sysout.println(" " + entry.getKey() + + ": \"" + entry.getValue() + "\""); + } + sysout.println(" ]"); + } + sysout.println(); + } return 0; } catch (Exception e) { syserr.println(analyzeException("resolve", e, argsList)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/JsonSerDeser.java index e086e36..b21a1d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/JsonSerDeser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/JsonSerDeser.java @@ -19,6 +19,7 @@ package org.apache.hadoop.registry.client.binding; import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataInputStream; @@ -75,15 +76,18 @@ */ public JsonSerDeser(Class classType, byte[] header) { Preconditions.checkArgument(classType != null, "null classType"); - Preconditions.checkArgument(header != null, "null header"); this.classType = classType; this.mapper = new ObjectMapper(); mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); // make an immutable copy to keep findbugs happy. - byte[] h = new byte[header.length]; - System.arraycopy(header, 0, h, 0, header.length); - this.header = h; + if (header != null) { + byte[] h = new byte[header.length]; + System.arraycopy(header, 0, h, 0, header.length); + this.header = h; + } else { + this.header = null; + } } public String getName() { @@ -234,6 +238,7 @@ private void writeJsonAsBytes(T instance, * @throws IOException */ public byte[] toByteswithHeader(T instance) throws IOException { + Preconditions.checkNotNull(header, "null header"); byte[] body = toBytes(instance); ByteBuffer buffer = ByteBuffer.allocate(body.length + header.length); @@ -246,6 +251,19 @@ private void writeJsonAsBytes(T instance, * Deserialize from a byte array * @param path path the data came from * @param bytes byte array + * @throws IOException all problems + * @throws EOFException not enough data + * @throws InvalidRecordException if the parsing failed -the record is invalid + */ + public T fromBytes(String path, byte[] bytes) throws IOException, + InvalidRecordException { + return fromBytes(path, bytes, 0, ""); + } + + /** + * Deserialize from a byte array with an offset + * @param path path the data came from + * @param bytes byte array * @return offset in the array to read from * @throws IOException all problems * @throws EOFException not enough data @@ -253,11 +271,33 @@ private void writeJsonAsBytes(T instance, */ public T fromBytes(String path, byte[] bytes, int offset) throws IOException, InvalidRecordException { + return fromBytes(path, bytes, offset, ""); + } + + /** + * Deserialize from a byte array with an offset + * @param path path the data came from + * @param bytes byte array + * @return offset in the array to read from + * @param marker an optional string which, if set, MUST be present in the + * UTF-8 parsed payload. + * @throws IOException all problems + * @throws EOFException not enough data + * @throws InvalidRecordException if the parsing failed -the record is invalid + */ + public T fromBytes(String path, byte[] bytes, int offset, String marker) + throws IOException, + InvalidRecordException { int data = bytes.length - offset; if (data <= 0) { throw new EOFException("No data at " + path); } String json = new String(bytes, offset, data, UTF_8); + if (StringUtils.isNotEmpty(marker) + && !json.contains(marker)) { + throw new InvalidRecordException(path, + "Missing marker string " + marker); + } try { return fromJson(json); } catch (JsonProcessingException e) { @@ -277,6 +317,8 @@ public T fromBytes(String path, byte[] bytes, int offset) throws IOException, */ @SuppressWarnings("unchecked") public T fromBytesWithHeader(String path, byte[] buffer) throws IOException { + Preconditions.checkNotNull(header, "null header"); + int hlen = header.length; int blen = buffer.length; if (hlen > 0) { @@ -324,4 +366,11 @@ public synchronized String toJson(T instance) throws IOException, return mapper.writeValueAsString(instance); } + public String toString(T instance) { + try { + return toJson(instance); + } catch (IOException e) { + return "failed to convert to a string: " + e; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java index b4254a3..9f38eac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryTypeUtils.java @@ -22,9 +22,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; -import org.apache.hadoop.registry.client.types.AddressTypes; +import static org.apache.hadoop.registry.client.types.AddressTypes.*; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; +import org.apache.hadoop.registry.client.types.ServiceRecord; import java.net.InetSocketAddress; import java.net.MalformedURLException; @@ -32,7 +33,9 @@ import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Static methods to work with registry types —primarily endpoints and the @@ -94,9 +97,9 @@ public static Endpoint inetAddrEndpoint(String api, Preconditions.checkArgument(protocolType != null, "null protocolType"); Preconditions.checkArgument(hostname != null, "null hostname"); return new Endpoint(api, - AddressTypes.ADDRESS_HOSTNAME_AND_PORT, + ADDRESS_HOSTNAME_AND_PORT, protocolType, - tuplelist(hostname, Integer.toString(port))); + hostnamePortPair(hostname, port)); } /** @@ -107,66 +110,55 @@ public static Endpoint inetAddrEndpoint(String api, * @param address the address as a tuple of (hostname, port) * @return the new endpoint */ - public static Endpoint ipcEndpoint(String api, - boolean protobuf, List address) { - ArrayList> addressList = new ArrayList>(); - if (address != null) { - addressList.add(address); - } + public static Endpoint ipcEndpoint(String api, InetSocketAddress address) { return new Endpoint(api, - AddressTypes.ADDRESS_HOSTNAME_AND_PORT, - protobuf ? ProtocolTypes.PROTOCOL_HADOOP_IPC_PROTOBUF - : ProtocolTypes.PROTOCOL_HADOOP_IPC, - addressList); + ADDRESS_HOSTNAME_AND_PORT, + ProtocolTypes.PROTOCOL_HADOOP_IPC, + address== null ? null: hostnamePortPair(address)); } /** - * Create a single-element list of tuples from the input. - * that is, an input ("a","b","c") is converted into a list - * in the form [["a","b","c"]] - * @param t1 tuple elements - * @return a list containing a single tuple + * Create a single entry map + * @param key map entry key + * @param val map entry value + * @return a 1 entry map. */ - public static List> tuplelist(String... t1) { - List> outer = new ArrayList>(); - outer.add(tuple(t1)); - return outer; + public static Map map(String key, String val) { + Map map = new HashMap(1); + map.put(key, val); + return map; } /** - * Create a tuples from the input. - * that is, an input ("a","b","c") is converted into a list - * in the form ["a","b","c"] - * @param t1 tuple elements - * @return a single tuple as a list + * Create a URI + * @param uri value + * @return a 1 entry map. */ - public static List tuple(String... t1) { - return Arrays.asList(t1); + public static Map uri(String uri) { + return map(ADDRESS_URI, uri); } - + /** - * Create a tuples from the input, converting all to Strings in the process - * that is, an input ("a", 7, true) is converted into a list - * in the form ["a","7,"true"] - * @param t1 tuple elements - * @return a single tuple as a list + * Create a hostname port address pair + * @param hostname + * @param port + * @return a 1 entry map. */ - public static List tuple(Object... t1) { - List l = new ArrayList(t1.length); - for (Object t : t1) { - l.add(t.toString()); - } - return l; + public static Map hostnamePortPair(String hostname, int port) { + Map map = + map(ADDRESS_HOSTNAME_FIELD, hostname); + map.put(ADDRESS_PORT_FIELD, Integer.toString(port)); + return map; } /** - * Convert a socket address pair into a string tuple, (host, port). - * TODO JDK7: move to InetAddress.getHostString() to avoid DNS lookups. - * @param address an address - * @return an element for the address list + * Create a hostname port address pair + * @param hostname + * @param port + * @return a 1 entry map. */ - public static List marshall(InetSocketAddress address) { - return tuple(address.getHostName(), address.getPort()); + public static Map hostnamePortPair(InetSocketAddress address) { + return hostnamePortPair(address.getHostName(), address.getPort()); } /** @@ -199,24 +191,28 @@ public static void requireAddressType(String required, Endpoint epr) throws if (epr == null) { return null; } - requireAddressType(AddressTypes.ADDRESS_URI, epr); - List> addresses = epr.addresses; + requireAddressType(ADDRESS_URI, epr); + List> addresses = epr.addresses; if (addresses.size() < 1) { throw new InvalidRecordException(epr.toString(), "No addresses in endpoint"); } List results = new ArrayList(addresses.size()); - for (List address : addresses) { - if (address.size() != 1) { - throw new InvalidRecordException(epr.toString(), - "Address payload invalid: wrong element count: " + - address.size()); - } - results.add(address.get(0)); + for (Map address : addresses) { + results.add(getAddressField(address, ADDRESS_URI)); } return results; } + public static String getAddressField(Map address, + String key) throws InvalidRecordException { + String val = address.get(key); + if (val == null) { + throw new InvalidRecordException("","missing address field " + key); + } + return val; + } + /** * Get the address URLs. Guranteed to return at least one address. * @param epr endpoint @@ -237,4 +233,54 @@ public static void requireAddressType(String required, Endpoint epr) throws } return results; } + + /** + * Validate the record by checking for null fields and other invalid + * conditions + * @param path path for exceptions + * @param record record to validate. May be null + * @throws InvalidRecordException on invalid entries + */ + public static void validateServiceRecord(String path, ServiceRecord record) + throws InvalidRecordException { + if (record == null) { + throw new InvalidRecordException(path, "Null record"); + } + if (!ServiceRecord.RECORD_TYPE.equals(record.type)) { + throw new InvalidRecordException(path, + "invalid record type field: \"" + record.type + "\""); + } + + if (record.external != null) { + for (Endpoint endpoint : record.external) { + validateEndpoint(path, endpoint); + } + } + if (record.internal != null) { + for (Endpoint endpoint : record.internal) { + validateEndpoint(path, endpoint); + } + } + } + + /** + * Validate the endpoint by checking for null fields and other invalid + * conditions + * @param path path for exceptions + * @param endpoint endpoint to validate. May be null + * @throws InvalidRecordException on invalid entries + */ + public static void validateEndpoint(String path, Endpoint endpoint) + throws InvalidRecordException { + if (endpoint == null) { + throw new InvalidRecordException(path, "Null endpoint"); + } + try { + endpoint.validate(); + } catch (RuntimeException e) { + throw new InvalidRecordException(path, e.toString()); + } + } + + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/exceptions/NoRecordException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/exceptions/NoRecordException.java index 160433f..f23c495 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/exceptions/NoRecordException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/exceptions/NoRecordException.java @@ -25,13 +25,8 @@ /** * Raised if there is no {@link ServiceRecord} resolved at the end - * of the specified path, for reasons such as: - *
    - *
  • There wasn't enough data to contain a Service Record.
  • - *
  • The start of the data did not match the {@link ServiceRecordHeader} - * header.
  • - *
- * + * of the specified path. + *

* There may be valid data of some form at the end of the path, but it does * not appear to be a Service Record. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java index c54c205..ccc0b57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java @@ -24,9 +24,11 @@ import org.apache.hadoop.registry.client.api.BindFlags; import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; import org.apache.hadoop.registry.client.binding.RegistryUtils; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException; +import org.apache.hadoop.registry.client.exceptions.NoRecordException; import org.apache.hadoop.registry.client.types.RegistryPathStatus; import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.zookeeper.CreateMode; @@ -103,10 +105,12 @@ public void bind(String path, int flags) throws IOException { Preconditions.checkArgument(record != null, "null record"); validatePath(path); + // validate the record before putting it + RegistryTypeUtils.validateServiceRecord(path, record); LOG.info("Bound at {} : {}", path, record); CreateMode mode = CreateMode.PERSISTENT; - byte[] bytes = serviceRecordMarshal.toByteswithHeader(record); + byte[] bytes = serviceRecordMarshal.toBytes(record); zkSet(path, mode, bytes, getClientAcls(), ((flags & BindFlags.OVERWRITE) != 0)); } @@ -114,7 +118,16 @@ public void bind(String path, @Override public ServiceRecord resolve(String path) throws IOException { byte[] bytes = zkRead(path); - return serviceRecordMarshal.fromBytesWithHeader(path, bytes); + + // filter out any records which are clearly invalid + // by being too short for the mandatory record type field + if (bytes.length < ServiceRecord.RECORD_TYPE.length()) { + throw new NoRecordException(path, "No record found"); + } + ServiceRecord record = serviceRecordMarshal.fromBytes(path, + bytes, 0, ServiceRecord.RECORD_TYPE); + RegistryTypeUtils.validateServiceRecord(path, record); + return record; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/AddressTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/AddressTypes.java index 192819c..36dbf0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/AddressTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/AddressTypes.java @@ -38,6 +38,8 @@ * */ public static final String ADDRESS_HOSTNAME_AND_PORT = "host/port"; + public static final String ADDRESS_HOSTNAME_FIELD = "host"; + public static final String ADDRESS_PORT_FIELD = "port"; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java index 51418d9..c2e243e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java @@ -21,14 +21,18 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.registry.client.binding.JsonSerDeser; import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; +import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Description of a single service/component endpoint. @@ -67,7 +71,7 @@ /** * a list of address tuples —tuples whose format depends on the address type */ - public List> addresses; + public List> addresses; /** * Create an empty instance. @@ -84,10 +88,11 @@ public Endpoint(Endpoint that) { this.api = that.api; this.addressType = that.addressType; this.protocolType = that.protocolType; - this.addresses = new ArrayList>(that.addresses.size()); - for (List address : addresses) { - List addr2 = new ArrayList(address.size()); - Collections.copy(address, addr2); + this.addresses = newAddresses(that.addresses.size()); + for (Map address : that.addresses) { + Map addr2 = new HashMap(address.size()); + addr2.putAll(address); + addresses.add(addr2); } } @@ -101,15 +106,54 @@ public Endpoint(Endpoint that) { public Endpoint(String api, String addressType, String protocolType, - List> addrs) { + List> addrs) { this.api = api; this.addressType = addressType; this.protocolType = protocolType; - this.addresses = new ArrayList>(); + this.addresses = newAddresses(0); if (addrs != null) { addresses.addAll(addrs); } } + + /** + * Build an endpoint with a list of addresses + * @param api API name + * @param addressType address type + * @param protocolType protocol type + * @param addrs addresses + */ + public Endpoint(String api, + String addressType, + String protocolType) { + this.api = api; + this.addressType = addressType; + this.protocolType = protocolType; + this.addresses = newAddresses(0); + } + + /** + * Build an endpoint with a list of addresses + * @param api API name + * @param addressType address type + * @param protocolType protocol type + * @param addrs addresses. Null elements will be skipped + */ + public Endpoint(String api, + String addressType, + String protocolType, + Map...addrs) { + this(api, addressType, protocolType); + for (Map addr : addrs) { + if (addr!=null) { + addresses.add(addr); + } + } + } + + private List> newAddresses(int i) { + return new ArrayList>(0); + } /** * Build an endpoint from a list of URIs; each URI @@ -125,40 +169,16 @@ public Endpoint(String api, this.addressType = AddressTypes.ADDRESS_URI; this.protocolType = protocolType; - List> addrs = new ArrayList>(uris.length); + List> addrs = newAddresses(uris.length); for (URI uri : uris) { - addrs.add(RegistryTypeUtils.tuple(uri.toString())); + addrs.add(RegistryTypeUtils.uri(uri.toString())); } this.addresses = addrs; } @Override public String toString() { - final StringBuilder sb = new StringBuilder("Endpoint{"); - sb.append("api='").append(api).append('\''); - sb.append(", addressType='").append(addressType).append('\''); - sb.append(", protocolType='").append(protocolType).append('\''); - - sb.append(", addresses="); - if (addresses != null) { - sb.append("[ "); - for (List address : addresses) { - sb.append("[ "); - if (address == null) { - sb.append("NULL entry in address list"); - } else { - for (String elt : address) { - sb.append('"').append(elt).append("\" "); - } - } - sb.append("] "); - }; - sb.append("] "); - } else { - sb.append("(null) "); - } - sb.append('}'); - return sb.toString(); + return marshalToString.toString(this); } /** @@ -173,7 +193,7 @@ public void validate() { Preconditions.checkNotNull(addressType, "null addressType field"); Preconditions.checkNotNull(protocolType, "null protocolType field"); Preconditions.checkNotNull(addresses, "null addresses field"); - for (List address : addresses) { + for (Map address : addresses) { Preconditions.checkNotNull(address, "null element in address"); } } @@ -184,7 +204,19 @@ public void validate() { * @throws CloneNotSupportedException */ @Override - protected Object clone() throws CloneNotSupportedException { + public Object clone() throws CloneNotSupportedException { return super.clone(); } + + + /** + * Static instance of service record marshalling + */ + private static class Marshal extends JsonSerDeser { + private Marshal() { + super(Endpoint.class, null); + } + } + + private static final Marshal marshalToString = new Marshal(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ProtocolTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ProtocolTypes.java index f225cf0..b836b00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ProtocolTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ProtocolTypes.java @@ -34,16 +34,11 @@ String PROTOCOL_FILESYSTEM = "hadoop/filesystem"; /** - * Classic Hadoop IPC : {@value}. + * Hadoop IPC, "classic" or protobuf : {@value}. */ String PROTOCOL_HADOOP_IPC = "hadoop/IPC"; /** - * Hadoop protocol buffers IPC: {@value}. - */ - String PROTOCOL_HADOOP_IPC_PROTOBUF = "hadoop/protobuf"; - - /** * Corba IIOP: {@value}. */ String PROTOCOL_IIOP = "IIOP"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java index 378127f..9403d31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; import org.codehaus.jackson.annotate.JsonAnyGetter; import org.codehaus.jackson.annotate.JsonAnySetter; import org.codehaus.jackson.map.annotate.JsonSerialize; @@ -41,6 +42,17 @@ public class ServiceRecord implements Cloneable { /** + * A type string which MUST be in the serialized json. This permits + * fast discarding of invalid entries + */ + public static final String RECORD_TYPE = "JSONServiceRecord"; + + /** + * The type field. This must be the string {@link #RECORD_TYPE} + */ + public String type = RECORD_TYPE; + + /** * Description string */ public String description; @@ -233,17 +245,5 @@ protected Object clone() throws CloneNotSupportedException { return super.clone(); } - /** - * Validate the record by checking for null fields and other invalid - * conditions - * @throws NullPointerException if a field is null when it - * MUST be set. - * @throws RuntimeException on invalid entries - */ - public void validate() { - for (Endpoint endpoint : external) { - Preconditions.checkNotNull("null endpoint", endpoint); - endpoint.validate(); - } - } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecordHeader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecordHeader.java index 2f75dba..4f3c863 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecordHeader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecordHeader.java @@ -28,6 +28,8 @@ @InterfaceAudience.Public @InterfaceStability.Evolving public class ServiceRecordHeader { + + /** * Header of a service record: "jsonservicerec" * By making this over 12 bytes long, we can auto-determine which entries diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.tla b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.tla index 1c19ade..69651f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.tla +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.tla @@ -4,6 +4,7 @@ EXTENDS FiniteSets, Sequences, Naturals, TLC (* +============================================================================ * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,6 +20,7 @@ EXTENDS FiniteSets, Sequences, Naturals, TLC * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. +============================================================================ *) (* @@ -71,13 +73,22 @@ CONSTANTS MknodeActions \* all possible mkdir actions +ASSUME PathChars \in STRING +ASSUME Paths \in STRING + +(* Data in records is JSON, hence a string *) +ASSUME Data \in STRING + +---------------------------------------------------------------------------------------- (* the registry*) VARIABLE registry + (* Sequence of actions to apply to the registry *) VARIABLE actions + ---------------------------------------------------------------------------------------- (* Tuple of all variables. *) @@ -92,7 +103,6 @@ vars == << registry, actions >> (* Persistence policy *) PersistPolicySet == { - "", \* Undefined; field not present. PERMANENT is implied. "permanent", \* persists until explicitly removed "application", \* persists until the application finishes "application-attempt", \* persists until the application attempt finishes @@ -104,7 +114,6 @@ TypeInvariant == /\ \A p \in PersistPolicies: p \in PersistPolicySet - ---------------------------------------------------------------------------------------- @@ -129,6 +138,14 @@ RegistryEntry == [ ] +(* Define the set of all string to string mappings *) + +StringMap == [ + STRING |-> STRING +] + + + (* An endpoint in a service record *) @@ -140,21 +157,14 @@ Endpoint == [ addresses: Addresses ] -(* Attributes are the set of all string to string mappings *) - -Attributes == [ -STRING |-> STRING -] - (* A service record *) ServiceRecord == [ - \* ID -used when applying the persistence policy - yarn_id: STRING, - \* the persistence policy - yarn_persistence: PersistPolicySet, + \* This MUST be present: if it is not then the data is not a service record + \* This permits shortcut scan & reject of byte arrays without parsing + type: "JSONServiceRecord", \*A description description: STRING, @@ -166,9 +176,34 @@ ServiceRecord == [ internal: Endpoints, \* Attributes are a function - attributes: Attributes + attributes: StringMap ] +---------------------------------------------------------------------------------------- + +(* + There is an operation serialize whose internals are not defined, + Which converts the service records to JSON + *) + +CONSTANT serialize(_) + +(* A function which returns true iff the byte stream is considered a valid service record. *) +CONSTANT containsServiceRecord(_) + +(* A function to deserialize a string to JSON *) +CONSTANT deserialize(_) + +ASSUME \A json \in STRING: containsServiceRecord(json) \in BOOLEAN + +(* Records can be serialized *) +ASSUME \A r \in ServiceRecord : serialize(r) \in STRING /\ containsServiceRecord(serialize(r)) + +(* All strings for which containsServiceRecord() holds can be deserialized *) +ASSUME \A json \in STRING: containsServiceRecord(json) => deserialize(json) \in ServiceRecord + + + ---------------------------------------------------------------------------------------- @@ -304,8 +339,8 @@ validRegistry(R) == \* an entry must be the root entry or have a parent entry /\ \A e \in R: isRootEntry(e) \/ exists(R, parent(e.path)) - \* If the entry has data, it must be a service record - /\ \A e \in R: (e.data = << >> \/ e.data \in ServiceRecords) + \* If the entry has data, it must contain a service record + /\ \A e \in R: (e.data = << >> \/ containsServiceRecord(e.data)) ---------------------------------------------------------------------------------------- @@ -336,13 +371,13 @@ mknode() adds a new empty entry where there was none before, iff *) mknodeSimple(R, path) == - LET record == [ path |-> path, data |-> <<>> ] + LET entry == [ path |-> path, data |-> <<>> ] IN \/ exists(R, path) - \/ (exists(R, parent(path)) /\ canBind(R, record) /\ (R' = R \union {record} )) + \/ (exists(R, parent(path)) /\ canBind(R, entry) /\ (R' = R \union {entry} )) (* -For all parents, the mknodeSimpl() criteria must apply. +For all parents, the mknodeSimple() criteria must apply. This could be defined recursively, though as TLA+ does not support recursion, an alternative is required @@ -350,7 +385,8 @@ an alternative is required Because this specification is declaring the final state of a operation, not the implemental, all that is needed is to describe those parents. -It declares that the mkdirSimple state applies to the path and all its parents in the set R' +It declares that the mknodeSimple() state applies to the path and all +its parents in the set R' *) mknodeWithParents(R, path) == @@ -402,7 +438,7 @@ purge(R, path, id, persistence) == => recursiveDelete(R, p2.path) (* -resolveRecord() resolves the record at a path or fails. +resolveEntry() resolves the record entry at a path or fails. It relies on the fact that if the cardinality of a set is 1, then the CHOOSE operator is guaranteed to return the single entry of that set, iff the choice predicate holds. @@ -411,19 +447,28 @@ Using a predicate of TRUE, it always succeeds, so this function selects the sole entry of the resolve operation. *) -resolveRecord(R, path) == +resolveEntry(R, path) == LET l == resolve(R, path) IN /\ Cardinality(l) = 1 /\ CHOOSE e \in l : TRUE (* + Resolve a record by resolving the entry and deserializing the result + *) +resolveRecord(R, path) == + deserialize(resolveEntry(R, path)) + + +(* The specific action of putting an entry into a record includes validating the record *) validRecordToBind(path, record) == \* The root entry must have permanent persistence - isRootPath(path) => (record.attributes["yarn:persistence"] = "permanent" - \/ record.attributes["yarn:persistence"] = "") + isRootPath(path) => ( + record.attributes["yarn:persistence"] = "permanent" + \/ record.attributes["yarn:persistence"] + \/ record.attributes["yarn:persistence"] = {}) (* @@ -432,13 +477,12 @@ marshalled as the data in the entry *) bindRecord(R, path, record) == /\ validRecordToBind(path, record) - /\ bind(R, [path |-> path, data |-> record]) + /\ bind(R, [path |-> path, data |-> serialize(record)]) ---------------------------------------------------------------------------------------- - (* The action queue can only contain one of the sets of action types, and by giving each a unique name, those sets are guaranteed to be disjoint diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/RegistryTestHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/RegistryTestHelper.java index 460ecad..fcb1a11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/RegistryTestHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/RegistryTestHelper.java @@ -46,11 +46,7 @@ import java.util.List; import java.util.Map; -import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.inetAddrEndpoint; -import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.ipcEndpoint; -import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.restEndpoint; -import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.tuple; -import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.webEndpoint; +import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.*; /** * This is a set of static methods to aid testing the registry operations. @@ -148,9 +144,9 @@ public static void validateEntry(ServiceRecord record) { assertEquals(API_WEBHDFS, webhdfs.api); assertEquals(AddressTypes.ADDRESS_URI, webhdfs.addressType); assertEquals(ProtocolTypes.PROTOCOL_REST, webhdfs.protocolType); - List> addressList = webhdfs.addresses; - List url = addressList.get(0); - String addr = url.get(0); + List> addressList = webhdfs.addresses; + Map url = addressList.get(0); + String addr = url.get("uri"); assertTrue(addr.contains("http")); assertTrue(addr.contains(":8020")); @@ -281,8 +277,8 @@ public static void addSampleEndpoints(ServiceRecord entry, String hostname) restEndpoint(API_WEBHDFS, new URI("http", hostname + ":8020", "/"))); - Endpoint endpoint = ipcEndpoint(API_HDFS, true, null); - endpoint.addresses.add(tuple(hostname, "8030")); + Endpoint endpoint = ipcEndpoint(API_HDFS, null); + endpoint.addresses.add(RegistryTypeUtils.hostnamePortPair(hostname, 8030)); entry.addInternalEndpoint(endpoint); InetSocketAddress localhost = new InetSocketAddress("localhost", 8050); entry.addInternalEndpoint( @@ -290,9 +286,7 @@ public static void addSampleEndpoints(ServiceRecord entry, String hostname) 8050)); entry.addInternalEndpoint( RegistryTypeUtils.ipcEndpoint( - IPC2, - true, - RegistryTypeUtils.marshall(localhost))); + IPC2, localhost)); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java index 14e3b1f..6e3a81d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java @@ -19,10 +19,11 @@ package org.apache.hadoop.registry.client.binding; import org.apache.hadoop.registry.RegistryTestHelper; +import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; import org.apache.hadoop.registry.client.exceptions.NoRecordException; import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.ServiceRecordHeader; import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; +import org.codehaus.jackson.JsonProcessingException; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; @@ -31,8 +32,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.EOFException; - /** * Test record marshalling */ @@ -44,6 +43,7 @@ public final Timeout testTimeout = new Timeout(10000); @Rule public TestName methodName = new TestName(); + private static RegistryUtils.ServiceRecordMarshal marshal; @BeforeClass @@ -55,12 +55,14 @@ public static void setupClass() { public void testRoundTrip() throws Throwable { String persistence = PersistencePolicies.PERMANENT; ServiceRecord record = createRecord(persistence); - record.set("customkey","customvalue"); - record.set("customkey2","customvalue2"); + record.set("customkey", "customvalue"); + record.set("customkey2", "customvalue2"); + RegistryTypeUtils.validateServiceRecord("", record); LOG.info(marshal.toJson(record)); byte[] bytes = marshal.toBytes(record); ServiceRecord r2 = marshal.fromBytes("", bytes, 0); assertMatches(record, r2); + RegistryTypeUtils.validateServiceRecord("", r2); } @Test @@ -69,7 +71,6 @@ public void testRoundTripHeaders() throws Throwable { byte[] bytes = marshal.toByteswithHeader(record); ServiceRecord r2 = marshal.fromBytesWithHeader("", bytes); assertMatches(record, r2); - } @Test(expected = NoRecordException.class) @@ -85,12 +86,26 @@ public void testUnmarshallHeaderTooShort() throws Throwable { marshal.fromBytesWithHeader("src", new byte[]{'a'}); } - @Test(expected = EOFException.class) + @Test(expected = InvalidRecordException.class) public void testUnmarshallNoBody() throws Throwable { - byte[] bytes = ServiceRecordHeader.getData(); - marshal.fromBytesWithHeader("src", bytes); + byte[] bytes = "this is not valid JSON at all and should fail".getBytes(); + marshal.fromBytes("src", bytes); } + @Test(expected = InvalidRecordException.class) + public void testUnmarshallWrongType() throws Throwable { + byte[] bytes = "{'type':''}".getBytes(); + ServiceRecord serviceRecord = marshal.fromBytes("marshalling", bytes); + RegistryTypeUtils.validateServiceRecord("validating", serviceRecord); + } + + @Test(expected = InvalidRecordException.class) + public void testUnmarshallNoType() throws Throwable { + byte[] bytes = "{}".getBytes(); + ServiceRecord serviceRecord = marshal.fromBytes("marshalling", + bytes, 0, ServiceRecord.RECORD_TYPE); + RegistryTypeUtils.validateServiceRecord("validating", serviceRecord); + } @Test public void testUnknownFieldsRoundTrip() throws Throwable { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java index 1cfb025..cce0f69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.registry.AbstractRegistryTest; import org.apache.hadoop.registry.client.api.BindFlags; +import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; import org.apache.hadoop.registry.client.binding.RegistryUtils; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.exceptions.NoRecordException; @@ -91,10 +92,8 @@ public void testLsParent() throws Throwable { childStats.values()); assertEquals(1, records.size()); ServiceRecord record = records.get(ENTRY_PATH); - assertNotNull(record); - record.validate(); + RegistryTypeUtils.validateServiceRecord(ENTRY_PATH, record); assertMatches(written, record); - } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md index a2a5009..ef8205f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md @@ -432,8 +432,8 @@ up according the lifecycle of that application. Description - addresses: List[List[String]] - a list of address tuples whose format depends on the address type + addresses: List[Map[String, String]] + a list of address maps addressType: String @@ -456,7 +456,7 @@ complex JSON structures in the text description. ### Field: Address Type -The addressType field defines the string format of entries. +The `addressType` field defines the string format of entries. Having separate types is that tools (such as a web viewer) can process binding strings without having to recognize the protocol. @@ -467,39 +467,42 @@ strings without having to recognize the protocol. binding format - `url` - `[URL]` + `uri` + {uri} `hostname` - `[hostname]` + `{hostname}` `inetaddress` - `[hostname, port]` + `{hostname, port}` `path` - `[/path/to/something]` + `{path}` `zookeeper` - `[quorum-entry, path]` + `{hostname, port, path}` -An actual zookeeper binding consists of a list of `hostname:port` bindings –the -quorum— and the path within. In the proposed schema, every quorum entry will be -listed as a triple of `[hostname, port, path]`. Client applications do not -expect the path to de be different across the quorum. The first entry in the -list of quorum hosts MUST define the path to be used by all clients. Later -entries SHOULD list the same path, though clients MUST ignore these. +In the zookeeper binding, every entry represents a single node in quorum, +the `hostname` and `port` fields defining the hostname of the ZK instance +and the port on which it is listening. The `path` field lists zookeeper path +for applications to use. For example, for HBase this would refer to the znode +containing information about the HBase cluster. + +The path MUST be identical across all address elements in the `addresses` list. +This ensures that any single address contains enough information to connect +to the quorum and connect to the relevant znode. New Address types may be defined; if not standard please prefix with the character sequence `"x-"`. -#### **Field: API** +#### **Field: "api""** APIs may be unique to a service class, or may be common across by service classes. They MUST be given unique names. These MAY be based on service @@ -524,12 +527,14 @@ overall application. It exports the URL to a load balancer. { "description" : "tomcat-based web application", - "registrationTime" : 1408638082444, "external" : [ { "api" : "www", "addressType" : "uri", "protocolType" : "REST", - "addresses" : [ [ "http://loadbalancer/" ] [ "http://loadbalancer2/" ] ] + "addresses" : [ + { "uri" : "http://loadbalancer/" }, + { "uri" : "http://loadbalancer2/" } + ] } ], "internal" : [ ] } @@ -545,21 +550,23 @@ will trigger the deletion of this entry /users/devteam/org-apache-tomcat/test1/components/container-1408631738011-0001-01-000001 { - "registrationTime" : 1408638082445, "yarn:id" : "container_1408631738011_0001_01_000001", - "yarn:persistence" : "3", - "description" : null, + "yarn:persistence" : "container", + "description" : "", "external" : [ { "api" : "www", "addressType" : "uri", "protocolType" : "REST", - "addresses" : [ [ "http://rack4server3:43572" ] ] + "addresses" : [{ "uri" : "rack4server3:43572" } ] } ], "internal" : [ { "api" : "jmx", "addressType" : "host/port", "protocolType" : "JMX", - "addresses" : [ [ "rack4server3", "43573" ] ] + "addresses" : [ { + "host" : "rack4server3", + "port" : "48551" + } ] } ] } @@ -571,7 +578,7 @@ external endpoint, the JMX addresses as internal. { "registrationTime" : 1408638082445, "yarn:id" : "container_1408631738011_0001_01_000002", - "yarn:persistence" : "3", + "yarn:persistence" : "container", "description" : null, "external" : [ { "api" : "www", @@ -583,7 +590,10 @@ external endpoint, the JMX addresses as internal. "api" : "jmx", "addressType" : "host/port", "protocolType" : "JMX", - "addresses" : [ [ "rack1server28", "35882" ] ] + "addresses" : [ { + "host" : "rack1server28", + "port" : "48551" + } ] } ] } @@ -887,3 +897,108 @@ Implementations may throttle update operations. **Rate of Polling** Clients which poll the registry may be throttled. + +# Complete service record example + +Below is a (non-normative) example of a service record retrieved +from a YARN application. + + + { + "type" : "JSONServiceRecord", + "description" : "Slider Application Master", + "external" : [ { + "api" : "org.apache.slider.appmaster", + "addressType" : "host/port", + "protocolType" : "hadoop/IPC", + "addresses" : [ { + "port" : "48551", + "host" : "nn.example.com" + } ] + }, { + "api" : "org.apache.http.UI", + "addressType" : "uri", + "protocolType" : "webui", + "addresses" : [ { + "uri" : "http://nn.example.com:40743" + } ] + }, { + "api" : "org.apache.slider.management", + "addressType" : "uri", + "protocolType" : "REST", + "addresses" : [ { + "uri" : "http://nn.example.com:40743/ws/v1/slider/mgmt" + } ] + }, { + "api" : "org.apache.slider.publisher", + "addressType" : "uri", + "protocolType" : "REST", + "addresses" : [ { + "uri" : "http://nn.example.com:40743/ws/v1/slider/publisher" + } ] + }, { + "api" : "org.apache.slider.registry", + "addressType" : "uri", + "protocolType" : "REST", + "addresses" : [ { + "uri" : "http://nn.example.com:40743/ws/v1/slider/registry" + } ] + }, { + "api" : "org.apache.slider.publisher.configurations", + "addressType" : "uri", + "protocolType" : "REST", + "addresses" : [ { + "uri" : "http://nn.example.com:40743/ws/v1/slider/publisher/slider" + } ] + }, { + "api" : "org.apache.slider.publisher.exports", + "addressType" : "uri", + "protocolType" : "REST", + "addresses" : [ { + "uri" : "http://nn.example.com:40743/ws/v1/slider/publisher/exports" + } ] + } ], + "internal" : [ { + "api" : "org.apache.slider.agents.secure", + "addressType" : "uri", + "protocolType" : "REST", + "addresses" : [ { + "uri" : "https://nn.example.com:52705/ws/v1/slider/agents" + } ] + }, { + "api" : "org.apache.slider.agents.oneway", + "addressType" : "uri", + "protocolType" : "REST", + "addresses" : [ { + "uri" : "https://nn.example.com:33425/ws/v1/slider/agents" + } ] + } ], + "yarn:persistence" : "application", + "yarn:id" : "application_1414052463672_0028" + } + +It publishes a number of endpoints, both internal and external. + +External: + +1. The IPC hostname and port for client-AM communications +1. URL to the AM's web UI +1. A series of REST URLs under the web UI for specific application services. +The details are irrelevant —note that they use an application-specific API +value to ensure uniqueness. + +Internal: +1. Two URLS to REST APIs offered by the AM for containers deployed by + the application itself. + +Python agents running in the containers retrieve the internal endpoint +URLs to communicate with their AM. The record is resolved on container startup +and cached until communications problems occur. At that point the registry is +queried for the current record, then an attempt is made to reconnect to the AM. + +Here "connectivity" problems means both "low level socket/IO errors" and +"failures in HTTPS authentication". The agents use two-way HTTPS authentication +—if the AM fails and another application starts listening on the same ports +it will trigger an authentication failure and hence service record reread. + +