diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java index 7115a4c..f4fecfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java @@ -144,6 +144,26 @@ String KEY_DNS_ZONES_DIR = DNS_PREFIX + "zones-dir"; /** + * Split Reverse Zone. + * It may be necessary to spit large reverse zone subnets + * into multiple zones to handle existing hosts collocated + * with containers. + */ + String KEY_DNS_SPLIT_REVERSE_ZONE = DNS_PREFIX + "split-reverse-zone"; + + /** + * Default value for splitting the reverse zone. + */ + boolean DEFAULT_DNS_SPLIT_REVERSE_ZONE = false; + + /** + * Split Reverse Zone IP Range. + * How many IPs should be part of each reverse zone split + */ + String KEY_DNS_SPLIT_REVERSE_ZONE_RANGE = DNS_PREFIX + + "split-reverse-zone-range"; + + /** * Key to set if the registry is secure: {@value}. * Turning it on changes the permissions policy from "open access" * to restrictions on kerberos with the option of diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java index 52b3c37..ff94afc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.registry.server.dns; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.commons.net.util.Base64; @@ -269,17 +270,102 @@ public boolean accept( } /** + * Return the number of zones in the map. + * + * @return number of zones in the map + */ + @VisibleForTesting + protected int getZoneCount() { + return zones.size(); + } + + /** * Initializes the reverse lookup zone (mapping IP to name). * * @param conf the Hadoop configuration. - * @throws IOException + * @throws IOException if the DNSSEC key can not be read. */ private void initializeReverseLookupZone(Configuration conf) throws IOException { - Name reverseLookupZoneName = getReverseZoneName(conf); - Zone reverseLookupZone = - configureZone(reverseLookupZoneName, conf); - zones.put(reverseLookupZone.getOrigin(), reverseLookupZone); + // Determine if the subnet should be split into + // multiple reverse zones, this can be necessary in + // network configurations where the hosts and containers + // are part of the same subnet (i.e. the containers only use + // part of the subnet). + Boolean shouldSplitReverseZone = + conf.getBoolean(KEY_DNS_SPLIT_REVERSE_ZONE, + DEFAULT_DNS_SPLIT_REVERSE_ZONE); + if(shouldSplitReverseZone) { + int subnetCount = getSubnetCountForReverseZones(conf); + addSplitReverseZones(conf, subnetCount); + // Single reverse zone + } else { + Name reverseLookupZoneName = getReverseZoneName(conf); + Zone reverseLookupZone = + configureZone(reverseLookupZoneName, conf); + zones.put(reverseLookupZone.getOrigin(), reverseLookupZone); + } + } + + /** + * When splitting the reverse zone, return the number of subnets needed, + * given the range and netmask. + * + * @param conf the Hadoop configuration. + * @return The number of subnets given the range and netmask. + */ + private int getSubnetCountForReverseZones(Configuration conf) { + String subnet = conf.get(KEY_DNS_ZONE_SUBNET); + String mask = conf.get(KEY_DNS_ZONE_MASK); + String range = conf.get(KEY_DNS_SPLIT_REVERSE_ZONE_RANGE); + + int parsedRange; + try { + parsedRange = Integer.parseInt(range); + } catch (NumberFormatException e) { + LOG.error("The supplied range is not a valid integer: Supplied range: ", + range); + throw e; + } + if(parsedRange<0) { + LOG.error("Range cannot be negative: Supplied range: ", parsedRange); + } + + int ipCount; + try { + SubnetUtils subnetUtils = new SubnetUtils(subnet, mask); + subnetUtils.setInclusiveHostCount(true); + ipCount = subnetUtils.getInfo().getAddressCount(); + + } catch (IllegalArgumentException e) { + LOG.error("The subnet or mask is invalid: Subnet: {} Mask: {}", subnet, + mask); + throw e; + } + return ipCount / parsedRange; + } + + /** + * Create the zones based on the zone count. + * + * @param conf the Hadoop configuration. + * @param subnetCount number of subnets to create reverse zones for. + * @throws IOException if the DNSSEC key can not be read. + */ + @VisibleForTesting + protected void addSplitReverseZones(Configuration conf, int subnetCount) + throws IOException { + String subnet = conf.get(KEY_DNS_ZONE_SUBNET); + String range = conf.get(KEY_DNS_SPLIT_REVERSE_ZONE_RANGE); + + // Add the split reverse zones + for(int idx=0; idx ipPartsOut = new ArrayList<>(); + // First octet + long temp = ipNum + range * (long) index; + ipPartsOut.add(0, temp / pow3); + + // Second octet + temp = temp - ipPartsOut.get(0) * pow3; + ipPartsOut.add(1, temp / pow2); + + // Third octet + temp = temp - ipPartsOut.get(1) * pow2; + ipPartsOut.add(2, temp / pow1); + + // Fourth octet + temp = temp - ipPartsOut.get(2) * pow1; + ipPartsOut.add(3, temp); + + return StringUtils.join(ipPartsOut, '.'); + } + + @VisibleForTesting + protected static long[] splitIp(String baseIp) throws UnknownHostException { + InetAddress inetAddress; + try { + inetAddress = InetAddress.getByName(baseIp); + } catch (UnknownHostException e) { + LOG.error("Base IP address is invalid"); + throw e; + } + if (inetAddress instanceof Inet6Address) { + throw new IllegalArgumentException("IPv6 is not yet supported for " + + "reverse zones"); + } + byte[] octets = inetAddress.getAddress(); + if (octets.length != 4) { + throw new IllegalArgumentException("Base IP address is invalid"); + } + long[] results = new long[4]; + for(int i=0; i